public static void SaveResultVideosLocally(List <SatyamResultsTableEntry> entries, string directoryName, int fps = 10) { directoryName = directoryName + "\\Raw"; if (!Directory.Exists(directoryName)) { Directory.CreateDirectory(directoryName); } SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess(); for (int i = 0; i < entries.Count; i++) { SatyamResultsTableEntry entry = entries[i]; SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString); MultiObjectTrackingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(task.jobEntry.JobParameters); string blobDir = URIUtilities.localDirectoryFullPathFromURI(task.SatyamURI); //VATIC_DVA_CrowdsourcedResult taskr = new VATIC_DVA_CrowdsourcedResult(satyamResult.TaskResult, task.SatyamURI, start, end, task.SatyamJobSubmissionsTableEntryID.ToString()); VATIC_DVA_CrowdsourcedResult taskr = MultiObjectTrackingAggregator.createVATIC_DVA_CrowdsourcedResultUsingSatyamBlobImageCount(satyamResult.TaskResult, blobDir, entry.ID.ToString(), job.FrameRate); MultiObjectTrackingResult res = taskr.getCompressedTracksInTimeSegment(); List <string> ImageURLs = satyamStorage.getURLListOfSpecificExtensionUnderSubDirectoryByURI(blobDir, new List <string>() { "jpg", "png" }); string videoName = URIUtilities.localDirectoryNameFromURI(blobDir) + "_" + URIUtilities.filenameFromURINoExtension(task.SatyamURI) + "_" + entry.ID; MultiObjectTrackingAnalyzer.generateVideoForEvaluation(ImageURLs, res, directoryName, videoName, job.FrameRate); } }
public static void SaveResultVideosLocally(List <SatyamResultsTableEntry> entries, string directoryName, int fps = 10) { directoryName = directoryName + "\\Raw"; if (!Directory.Exists(directoryName)) { Directory.CreateDirectory(directoryName); } SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess(); for (int i = 0; i < entries.Count; i++) { SatyamResultsTableEntry entry = entries[i]; SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString); MultiObjectTrackingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(task.jobEntry.JobParameters); //VATIC_DVA_CrowdsourcedResult taskr = new VATIC_DVA_CrowdsourcedResult(satyamResult.TaskResult, task.SatyamURI, start, end, task.SatyamJobSubmissionsTableEntryID.ToString()); VATIC_DVA_CrowdsourcedResult taskr = MultiObjectTrackingAggregator.createVATIC_DVA_CrowdsourcedResultUsingSatyamBlobImageCount(satyamResult.TaskResult, task.SatyamURI, entry.ID.ToString(), job.FrameRate); MultiObjectTrackingResult res = taskr.getCompressedTracksInTimeSegment(); List <string> ImageURLs = satyamStorage.getURLListOfSubDirectoryByURL(task.SatyamURI); string[] names = task.SatyamURI.Split('/'); string videoName = names[names.Length - 2] + "_" + entry.ID; generateVideoForEvaluation(ImageURLs, res, directoryName, videoName, job.FrameRate); } }
//public static string GetAggregatedResultString(List<SatyamResultsTableEntry> results, // int MinResults = TaskConstants.OBJECT_COUNTING_MTURK_MIN_RESULTS_TO_AGGREGATE, // int MaxResults = TaskConstants.OBJECT_COUNTING_MTURK_MAX_RESULTS_TO_AGGREGATE, // double MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND = TaskConstants.OBJECT_COUNTING_MTURK_MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND, // double MAX_DEVIATION_FRACTION = TaskConstants.OBJECT_COUNTING_MTURK_MAX_DEVIATION_FRACTION, // double SUPER_MAJORITY_VALUE = TaskConstants.OBJECT_COUNTING_MTURK_SUPER_MAJORITY_VALUE) //{ // string resultString = null; // List<ObjectCountingResult> resultList = new List<ObjectCountingResult>(); // foreach (SatyamResultsTableEntry entry in results) // { // SatyamResult res = JSonUtils.ConvertJSonToObject<SatyamResult>(entry.ResultString); // ObjectCountingResult taskr = JSonUtils.ConvertJSonToObject<ObjectCountingResult>(res.TaskResult); // resultList.Add(taskr); // } // ObjectCountingAggregatedResult r = getAggregatedResult(resultList, MinResults, MaxResults, MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND, MAX_DEVIATION_FRACTION, SUPER_MAJORITY_VALUE); // if (r != null) // { // string rString = JSonUtils.ConvertObjectToJSon<ObjectCountingAggregatedResult>(r); // SatyamAggregatedResult aggResult = new SatyamAggregatedResult(); // aggResult.SatyamTaskTableEntryID = results[0].SatyamTaskTableEntryID; // aggResult.AggregatedResultString = rString; // SatyamResult res = JSonUtils.ConvertJSonToObject<SatyamResult>(results[0].ResultString); // aggResult.TaskParameters = res.TaskParametersString; // resultString = JSonUtils.ConvertObjectToJSon<SatyamAggregatedResult>(aggResult); // } // return resultString; //} public static bool IsAcceptable(SatyamAggregatedResultsTableEntry aggResult, SatyamResultsTableEntry result, double MAX_ABSOLUTE_COUNT_DEVIATION = TaskConstants.OBJECT_COUNTING_MTURK_MAX_ABSOLUTE_COUNT_DEVIATION_FOR_PAYMENT, double MAX_DEVIATION_FRACTION = TaskConstants.OBJECT_COUNTING_MTURK_MAX_DEVIATION_FRACTION_FOR_PAYMENT) { SatyamResult res = JSonUtils.ConvertJSonToObject <SatyamResult>(result.ResultString); SatyamAggregatedResult aggres = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(aggResult.ResultString); ObjectCountingResult r = JSonUtils.ConvertJSonToObject <ObjectCountingResult>(res.TaskResult); ObjectCountingAggregatedResult ragg = JSonUtils.ConvertJSonToObject <ObjectCountingAggregatedResult>(aggres.AggregatedResultString); double maxdev = MAX_ABSOLUTE_COUNT_DEVIATION; if (ragg.Count * MAX_DEVIATION_FRACTION > maxdev) { maxdev = ragg.Count * MAX_DEVIATION_FRACTION; } if (Math.Abs(ragg.Count - r.Count) <= maxdev) { return(true); } else { return(false); } }
public static void SaveKITTIResultVideosLocally(List <SatyamResultsTableEntry> entries, string directoryName, int fps = 10) { directoryName = directoryName + "\\Raw"; if (!Directory.Exists(directoryName)) { Directory.CreateDirectory(directoryName); } SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess(); for (int i = 0; i < entries.Count; i++) { SatyamResultsTableEntry entry = entries[i]; SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString); MultiObjectTrackingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(task.jobEntry.JobParameters); //VATIC_DVA_CrowdsourcedResult taskr = VATIC_DVA_CrowdsourcedResult.createVATIC_DVA_CrowdsourcedResultUsingSatyamBlobImageCount(satyamResult.TaskResult, task.SatyamURI, task.SatyamJobSubmissionsTableEntryID.ToString(), job.FrameRate); //MultiObjectTrackingResult res = taskr.getCompressedTracksInTimeSegment(); string[] names = task.SatyamURI.Split('/'); string videoName = names[names.Length - 2] + "_" + entry.ID; string[] fields = videoName.Split('_'); string videoSequence = fields[fields.Length - 5]; int startingFrame = Convert.ToInt32(fields[fields.Length - 2]); int maxChunkEndFrame = startingFrame + job.ChunkDuration * job.FrameRate; int noFrameOverlap = (int)(job.ChunkOverlap * job.FrameRate); if (startingFrame != 0) { startingFrame -= noFrameOverlap; } if (entry.JobGUID == "1e43a983-548d-4a2e-8161-5537eb985902") { videoSequence = getCorrectSequenceNo(videoSequence); } string videoFrameDir = DirectoryConstants.KITTITrackingImages + videoSequence; List <string> files = Directory.GetFiles(videoFrameDir).ToList(); List <string> ImageURLs = new List <string>(); for (int j = startingFrame; j < files.Count && j < maxChunkEndFrame; j++) { ImageURLs.Add(files[j]); } if (ImageURLs.Count == 0) { continue; } VATIC_DVA_CrowdsourcedResult taskr = new VATIC_DVA_CrowdsourcedResult(satyamResult.TaskResult, videoName, entry.ID.ToString(), ImageURLs.Count, job.FrameRate); MultiObjectTrackingResult res = taskr.getCompressedTracksInTimeSegment(); generateVideoForEvaluation(ImageURLs, res, directoryName, videoName, job.FrameRate); } }
public static void SaveResultOnlyText(string jobGUID) { string directoryName = DirectoryConstants.defaultTempDirectory + "\\" + jobGUID + "\\"; SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); List <SatyamResultsTableEntry> entries = resultsDB.getEntriesByGUID(jobGUID); resultsDB.close(); if (!Directory.Exists(directoryName)) { Directory.CreateDirectory(directoryName); } directoryName = directoryName + "\\Raw\\"; if (!Directory.Exists(directoryName)) { Directory.CreateDirectory(directoryName); } for (int i = 0; i < entries.Count; i++) { SatyamResultsTableEntry entry = entries[i]; SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString); SatyamJob job = task.jobEntry; string result = satyamResult.TaskResult; string ofilename = URIUtilities.filenameFromURI(task.SatyamURI); string[] fields = ofilename.Split('.'); string fileName = ""; for (int j = 0; j < fields.Count(); j++) { fileName += fields[j]; if (j == fields.Count() - 1) { break; } fileName += "."; } //string fileName = ofilename; fileName = fileName + "-Result"; fileName = fileName + "-" + entry.ID; Console.WriteLine("Saving " + fileName); string resultFile = directoryName + fileName + ".txt"; StreamWriter f = new System.IO.StreamWriter(resultFile); f.WriteLine(result); f.Close(); } }
public static void SaveResultImagesLocally(List <SatyamResultsTableEntry> entries, string directoryName) { if (!Directory.Exists(directoryName)) { Directory.CreateDirectory(directoryName); } directoryName = directoryName + "\\Raw\\"; if (!Directory.Exists(directoryName)) { Directory.CreateDirectory(directoryName); } for (int i = 0; i < entries.Count; i++) { SatyamResultsTableEntry entry = entries[i]; SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString); SatyamJob job = task.jobEntry; string result = satyamResult.TaskResult; Image originalImage = ImageUtilities.getImageFromURI(task.SatyamURI); Image ResultImage = DrawResultStringOnImage(result, originalImage); string ofilename = URIUtilities.filenameFromURI(task.SatyamURI); string[] fields = ofilename.Split('.'); string fileName = ""; for (int j = 0; j < fields.Count(); j++) { fileName += fields[j]; if (j == fields.Count() - 1) { break; } fileName += "."; } //string fileName = ofilename; fileName = fileName + "-Result"; fileName = fileName + "-" + entry.ID; Console.WriteLine("Saving " + fileName); ImageUtilities.saveImage(ResultImage, directoryName, fileName); string resultFile = directoryName + fileName + ".txt"; StreamWriter f = new System.IO.StreamWriter(resultFile); f.WriteLine(result); f.Close(); } }
public static bool IsAcceptable(SatyamAggregatedResultsTableEntry aggResult, SatyamResultsTableEntry result) { SatyamResult res = JSonUtils.ConvertJSonToObject <SatyamResult>(result.ResultString); SatyamAggregatedResult aggres = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(aggResult.ResultString); SingleObjectLabelingResult r = JSonUtils.ConvertJSonToObject <SingleObjectLabelingResult>(res.TaskResult); SingleObjectLabelingAggregatedResult ragg = JSonUtils.ConvertJSonToObject <SingleObjectLabelingAggregatedResult>(aggres.AggregatedResultString); if (r.Category == ragg.Category) { return(true); } else { return(false); } }
private MultiObjectLocalizationAndLabelingResult LoadLatestTurkerResult(SatyamTaskTableEntry entry) { SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); List <SatyamResultsTableEntry> entries = resultsDB.getEntriesByGUIDAndTaskID(entry.JobGUID, entry.ID); if (entries.Count == 0) { return(null); } //organized sequentially SatyamResultsTableEntry prevResult = entries[entries.Count - 1]; Hidden_PrevResultID.Value = prevResult.ID.ToString(); SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(prevResult.ResultString); MultiObjectLocalizationAndLabelingResult res = JSonUtils.ConvertJSonToObject <MultiObjectLocalizationAndLabelingResult>(satyamResult.TaskResult); return(res); }
public static MultiObjectLabelingAndLocalizationAnalysisPerJob Analyse(List <SatyamResultsTableEntry> entries) { MultiObjectLabelingAndLocalizationAnalysisPerJob ana = new MultiObjectLabelingAndLocalizationAnalysisPerJob(); ana.jobTemplateType = entries[0].JobTemplateType; for (int i = 0; i < entries.Count; i++) { SatyamResultsTableEntry entry = entries[i]; SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString); SatyamJob job = task.jobEntry; MultiObjectLocalizationAndLabelingResult res = JSonUtils.ConvertJSonToObject <MultiObjectLocalizationAndLabelingResult>(satyamResult.TaskResult); } return(ana); }
public static void TestGetGlobalStatistics() { Dictionary <string, int> workerSubmissions = new Dictionary <string, int>(); Dictionary <string, int> HitSubmissions = new Dictionary <string, int>(); Dictionary <int, int> TaskSubmissions = new Dictionary <int, int>(); SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); List <SatyamResultsTableEntry> entries = new List <SatyamResultsTableEntry>(); entries = (resultsDB.getAllEntries()); for (int i = 0; i < entries.Count; i++) { SatyamResultsTableEntry entry = entries[i]; SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString); SatyamJob job = task.jobEntry; string worker = satyamResult.amazonInfo.WorkerID; string hitID = satyamResult.amazonInfo.HITID; int taskID = entry.SatyamTaskTableEntryID; if (!workerSubmissions.ContainsKey(worker)) { workerSubmissions.Add(worker, 0); } workerSubmissions[worker]++; if (!HitSubmissions.ContainsKey(hitID)) { HitSubmissions.Add(hitID, 0); } HitSubmissions[hitID]++; if (!TaskSubmissions.ContainsKey(taskID)) { TaskSubmissions.Add(taskID, 0); } TaskSubmissions[taskID]++; } Console.WriteLine(workerSubmissions.Count); Console.WriteLine(HitSubmissions.Count); Console.WriteLine(TaskSubmissions.Count); }
public SatyamResultSaveDataRequester(SatyamResultsTableEntry entry) { string status = entry.Status; if (status == ResultStatus.accepted || status == ResultStatus.accepted_Paid || status == ResultStatus.accepted_NotPaid) { resultAccepted = true; } else { resultAccepted = false; } if (status == ResultStatus.rejected_Paid || status == ResultStatus.accepted_Paid) { resultPaid = true; } else { resultPaid = false; } SatyamResult result = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); TaskStartTime = result.TaskStartTime; TaskEndTime = result.TaskEndTime; JobGUID = entry.JobGUID; JobTemplateType = entry.JobTemplateType; SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(result.TaskParametersString); //OrignialURI = task.OriginalURI; SatyamJob job = task.jobEntry; UserID = job.UserID; JobSubmitTime = job.JobSubmitTime; JobParameters = job.JobParameters; amazonInfo = new AmazonSavingInfo(); amazonInfo.AssignmentID = result.amazonInfo.AssignmentID; amazonInfo.pricePerHIT = result.amazonInfo.PricePerHIT; amazonInfo.TasksPerHIT = job.TasksPerJob; amazonInfo.WorkerID = result.amazonInfo.WorkerID; ResultString = result.TaskResult; }
public static bool IsAcceptable(SatyamAggregatedResultsTableEntry aggEntry, SatyamResultsTableEntry result) { switch (result.JobTemplateType) { case TaskConstants.Classification_Image: case TaskConstants.Classification_Image_MTurk: case TaskConstants.Classification_Video: case TaskConstants.Classification_Video_MTurk: return(SingleObjectLabelingAggregator.IsAcceptable(aggEntry, result)); case TaskConstants.Counting_Image: case TaskConstants.Counting_Image_MTurk: case TaskConstants.Counting_Video: case TaskConstants.Counting_Video_MTurk: return(ObjectCountingAggregator.IsAcceptable(aggEntry, result)); case TaskConstants.Detection_Image: case TaskConstants.Detection_Image_MTurk: return(MultiObjectLocalizationAndLabelingAggregator.IsAcceptable(aggEntry, result)); case TaskConstants.Tracking: case TaskConstants.Tracking_MTurk: return(MultiObjectTrackingAggregator.IsAcceptable(aggEntry, result)); case TaskConstants.Segmentation_Image: case TaskConstants.Segmentation_Image_MTurk: return(ImageSegmentationAggregator.IsAcceptable(aggEntry, result)); case TaskConstants.OpenEndedQuestion_Image: case TaskConstants.OpenEndedQuestion_Image_MTurk: return(true); //TBD case TaskConstants.TrackletLabeling: case TaskConstants.TrackletLabeling_MTurk: return(TrackletLabelingAggregator.IsAcceptable(aggEntry, result)); } return(false); }
public static SatyamResultPerJobDataAnalysis AnalyzeFromResultsTable(List <string> jobGUIDs) { Console.WriteLine("Analyzing Results Statistics of {0}", jobGUIDs[0]); SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); List <SatyamResultsTableEntry> entries = new List <SatyamResultsTableEntry>(); foreach (string guid in jobGUIDs) { entries.AddRange(resultsDB.getEntriesByGUIDOrderByID(guid)); } resultsDB.close(); if (entries.Count == 0) { return(null); } SatyamResultPerJobDataAnalysis ana = new SatyamResultPerJobDataAnalysis(); ana.JobGUID = entries[0].JobGUID; ana.totalResults = entries.Count; for (int i = 0; i < entries.Count; i++) { SatyamResultsTableEntry entry = entries[i]; SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString); SatyamJob job = task.jobEntry; if (ana.job == null) { ana.job = job; } ana.jobSubmitTime = job.JobSubmitTime; double moneyPerResult = 0; if (job.TasksPerJob > 0) { moneyPerResult = (double)satyamResult.amazonInfo.PricePerHIT / (double)job.TasksPerJob; } ana.PricePerResult.Add(entry.ID, moneyPerResult); int taskEntryID = entry.SatyamTaskTableEntryID; if (!ana.noResultsPerTask.ContainsKey(taskEntryID)) { ana.noResultsPerTask.Add(taskEntryID, 0); ana.noAcceptedResultsPerTask.Add(taskEntryID, 0); ana.noPaidResultsPerTask.Add(taskEntryID, 0); ana.moneyPaidPerTask.Add(taskEntryID, 0); ana.finalTaskEndTime.Add(taskEntryID, entry.SubmitTime); ana.ResultsPerWorkerPerTask.Add(taskEntryID, new Dictionary <string, List <string> >()); ana.noDifferentDuplicateResultsPerWorkerPerTask.Add(taskEntryID, new Dictionary <string, int>()); ana.taskSatyamUri.Add(taskEntryID, task.SatyamURI); } ana.noResultsPerTask[taskEntryID]++; ana.SequenceNumberOfResultPerTask.Add(ana.noResultsPerTask[taskEntryID]); if (entry.Status == ResultStatus.accepted || entry.Status == ResultStatus.accepted_Paid || entry.Status == ResultStatus.accepted_NotPaid) { ana.noAcceptedResultsPerTask[taskEntryID]++; ana.totalAcceptedResults++; } if (entry.Status == ResultStatus.accepted_Paid || entry.Status == ResultStatus.rejected_Paid) { ana.noPaidResultsPerTask[taskEntryID]++; ana.totalpaidResults++; ana.moneyPaidPerTask[taskEntryID] += moneyPerResult; ana.totalMoneyPaid += moneyPerResult; } if (ana.finalTaskEndTime[taskEntryID] < entry.SubmitTime) { ana.finalTaskEndTime[taskEntryID] = entry.SubmitTime; } if (ana.lastTaskEndTime < entry.SubmitTime) { ana.lastTaskEndTime = entry.SubmitTime; } TimeSpan dt = satyamResult.TaskEndTime - satyamResult.TaskStartTime; ana.timeTakenPerResult.Add(dt.TotalSeconds); if (!ana.AcceptedTimePerSequenceNumber.ContainsKey(ana.noResultsPerTask[taskEntryID])) { ana.AcceptedTimePerSequenceNumber.Add(ana.noResultsPerTask[taskEntryID], new List <double>()); ana.TimePerSequenceNumber.Add(ana.noResultsPerTask[taskEntryID], new List <double>()); } //ana.acceptedTimeTakenPerResultOrganizedByResultSequencePerTask[ana.noResultsPerTask[taskEntryID]].Add(dt.TotalSeconds); ana.TimePerSequenceNumber[ana.noResultsPerTask[taskEntryID]].Add(dt.TotalSeconds); if (!ana.AcceptedTimePerTask.ContainsKey(taskEntryID)) { ana.AcceptedTimePerTask.Add(taskEntryID, new List <double>()); ana.TimePerTask.Add(taskEntryID, new List <double>()); } //ana.AcceptedTimePerTask[taskEntryID].Add(dt.TotalSeconds); ana.TimePerTask[taskEntryID].Add(dt.TotalSeconds); if (ana.lastTaskEndTime < satyamResult.TaskEndTime) { ana.lastTaskEndTime = satyamResult.TaskEndTime; } if (!ana.finalHITEndTime.ContainsKey(satyamResult.amazonInfo.HITID)) { ana.finalHITEndTime.Add(satyamResult.amazonInfo.HITID, (entry.SubmitTime - job.JobSubmitTime).TotalSeconds); } if (ana.finalHITEndTime[satyamResult.amazonInfo.HITID] < (entry.SubmitTime - job.JobSubmitTime).TotalSeconds) { ana.finalHITEndTime[satyamResult.amazonInfo.HITID] = (entry.SubmitTime - job.JobSubmitTime).TotalSeconds; } if (entry.Status == ResultStatus.accepted || entry.Status == ResultStatus.accepted_Paid || entry.Status == ResultStatus.accepted_NotPaid) { ana.acceptedTimeTakenPerResult.Add(dt.TotalSeconds); ana.AcceptedTimePerTask[taskEntryID].Add(dt.TotalSeconds); ana.AcceptedTimePerSequenceNumber[ana.noResultsPerTask[taskEntryID]].Add(dt.TotalSeconds); } ana.resultArrivalTimes.Add((entry.SubmitTime - job.JobSubmitTime).TotalSeconds); if (!ana.noJobsPerWorker.ContainsKey(satyamResult.amazonInfo.WorkerID)) { ana.noJobsPerWorker.Add(satyamResult.amazonInfo.WorkerID, 0); ana.noAcceptedJobsPerWorker.Add(satyamResult.amazonInfo.WorkerID, 0); ana.noPaidJobsPerWorker.Add(satyamResult.amazonInfo.WorkerID, 0); } ana.noJobsPerWorker[satyamResult.amazonInfo.WorkerID]++; ana.SequenceNumberOfResultPerWorker.Add(ana.noJobsPerWorker[satyamResult.amazonInfo.WorkerID]); if (entry.Status == ResultStatus.accepted || entry.Status == ResultStatus.accepted_Paid || entry.Status == ResultStatus.accepted_NotPaid) { ana.noAcceptedJobsPerWorker[satyamResult.amazonInfo.WorkerID]++; } if (entry.Status == ResultStatus.accepted_Paid || entry.Status == ResultStatus.rejected_Paid) { ana.noPaidJobsPerWorker[satyamResult.amazonInfo.WorkerID]++; } if (!ana.ResultsPerWorkerPerTask[taskEntryID].ContainsKey(satyamResult.amazonInfo.WorkerID)) { ana.ResultsPerWorkerPerTask[taskEntryID].Add(satyamResult.amazonInfo.WorkerID, new List <string>()); ana.noDifferentDuplicateResultsPerWorkerPerTask[taskEntryID].Add(satyamResult.amazonInfo.WorkerID, 0); } else { // this worker has done this task before ana.noDuplicateResultsFromSameWorker++; } if (!ana.ResultsPerWorkerPerTask[taskEntryID][satyamResult.amazonInfo.WorkerID].Contains(satyamResult.TaskResult)) { ana.noDifferentDuplicateResultsPerWorkerPerTask[taskEntryID][satyamResult.amazonInfo.WorkerID]++; if (ana.noDifferentDuplicateResultsPerWorkerPerTask[taskEntryID][satyamResult.amazonInfo.WorkerID] > 1) { ana.noDecisionChangeAmongDuplicateResultsOfSameWorkerSameTask++; } } ana.ResultsPerWorkerPerTask[taskEntryID][satyamResult.amazonInfo.WorkerID].Add(satyamResult.TaskResult); ///////////////// imagenet only //List<SingleObjectLabelingResult> CurrentResultsOfThisTask = new List<SingleObjectLabelingResult>(); //foreach(string worker in ana.ResultsPerWorkerPerTask[taskEntryID].Keys) //{ // foreach(string workerResult in ana.ResultsPerWorkerPerTask[taskEntryID][worker]) // { // CurrentResultsOfThisTask.Add(JSonUtils.ConvertJSonToObject<SingleObjectLabelingResult>(workerResult)); // } //} //if (SingleObjectLabelingAggregator.getAggregatedResult(CurrentResultsOfThisTask) != null) //{ // // aggregation happened // if (!ana.timeTakenTillAggregationPerTask.ContainsKey(taskEntryID)) // { // // for the first time aggregated // ana.timeTakenTillAggregationPerTask.Add(taskEntryID, (entry.SubmitTime - job.JobSubmitTime).TotalSeconds); // } //} /////////////////////// } ana.resultArrivalTimes.Sort(); ana.noTasks = ana.noResultsPerTask.Count; ana.timeTakenPerResult.Sort(); ana.medianTimeTakenPerResultInSeconds = ana.timeTakenPerResult[ana.timeTakenPerResult.Count / 2]; if (ana.acceptedTimeTakenPerResult.Count != 0) { ana.acceptedTimeTakenPerResult.Sort(); ana.medianAcceptedTimeTakenPerResultInSeconds = ana.acceptedTimeTakenPerResult[ana.acceptedTimeTakenPerResult.Count / 2]; } ana.totalTimeTakenForJob = ana.lastTaskEndTime - ana.jobSubmitTime; ///////////////////////////////////// Per Task Analysis ////////////////////////////////////// ana.resultsPerTaskHistogram = new SortedDictionary <int, int>(); ana.resultsAcceptedPerTaskHistogram = new SortedDictionary <int, int>(); ana.resultsPaidPerTaskHistogram = new SortedDictionary <int, int>(); ana.moneyPaidPerTaskHistogram = new SortedDictionary <int, int>(); //cents ana.ResultsPerWorkerPerTaskHistogram = new SortedDictionary <int, int>(); int noAggregatedTasks = 0; List <int> taskIDs = ana.noResultsPerTask.Keys.ToList(); foreach (int taskID in taskIDs) { if (!ana.resultsPerTaskHistogram.ContainsKey(ana.noResultsPerTask[taskID])) { ana.resultsPerTaskHistogram.Add(ana.noResultsPerTask[taskID], 0); } ana.resultsPerTaskHistogram[ana.noResultsPerTask[taskID]]++; if (!ana.resultsAcceptedPerTaskHistogram.ContainsKey(ana.noAcceptedResultsPerTask[taskID])) { ana.resultsAcceptedPerTaskHistogram.Add(ana.noAcceptedResultsPerTask[taskID], 0); } ana.resultsAcceptedPerTaskHistogram[ana.noAcceptedResultsPerTask[taskID]]++; if (!ana.resultsPaidPerTaskHistogram.ContainsKey(ana.noPaidResultsPerTask[taskID])) { ana.resultsPaidPerTaskHistogram.Add(ana.noPaidResultsPerTask[taskID], 0); } ana.resultsPaidPerTaskHistogram[ana.noPaidResultsPerTask[taskID]]++; int moneyPaid = (int)Math.Floor(ana.moneyPaidPerTask[taskID] * 100); if (!ana.moneyPaidPerTaskHistogram.ContainsKey(moneyPaid)) { ana.moneyPaidPerTaskHistogram.Add(moneyPaid, 0); } ana.moneyPaidPerTaskHistogram[moneyPaid]++; if (ana.ResultsPerWorkerPerTask[taskID].Count != ana.noResultsPerTask[taskID]) { //has multiple results from same turker ana.noTasksWithDuplicateResultsFromSameWorker++; } if (ana.noPaidResultsPerTask[taskID] != 0) { noAggregatedTasks++; } } ana.avgNoPaidResultsPerTask = (double)ana.totalpaidResults / (double)noAggregatedTasks; //////////// duplicate analysis //Console.WriteLine("DuplicateResultsHistogram"); //foreach (int no in ana.ResultsPerWorkerPerTaskHistogram.Keys) //{ // Console.WriteLine("{0}, {1}", no, ana.ResultsPerWorkerPerTaskHistogram[no]); //} //Console.WriteLine("{0} images (in total {1}({2})) has duplicate results from same turker, \n" + // "\t{3} images have >=1 workers making more than majority number of results.\n" + // "\t{11} images aggregation is changed ({12} of which incorrectly) by duplicate results from same turker.\n" + // "\t{4} images(In total {5} times) with duplicate results include mixed(>= 2) decisions from same worker.\n" + // //"\t\t{6} images has correct decision among duplicate decisions.\n" + // //"\t\t{7} images({8} times) a worker has switched from incorrect to correct choice\n" + // //"\t\t\t{9} images({10} times) of which maintained the correct choice till the last time of their job on that image.", // ana.noTasksWithDuplicateResultsFromSameWorker, ana.noDuplicateResultsFromSameWorker, 0, // ana.noTaskWithDuplicateResultsAsMajority, // ana.noTasksWithMixedResultsFromSameWorker,ana.noDecisionChangeAmongDuplicateResultsOfSameWorkerSameTask, // ana.noTasksWithCorrectDuplicateResultsOfSameWorkerSameTask // //ana.noTasksWithWorkerSwitchedToCorrect, ana.noWorkerSwitchedToCorrect, // //ana.noTasksWithWorkerSwitchedToCorrectAndMaintained, ana.noWorkerSwitchedToCorrectAndMaintained, // //ana.noTaskWhoseDuplicateResultsChangedAggregation, ana.noTaskWhoseDuplicateResultsChangedAggregationIncorrectly //); ////////////////////////////// string outputDir = DirectoryConstants.defaultTempDirectory + jobGUIDs[0] + "\\ResultStatistics\\"; if (!Directory.Exists(outputDir)) { Directory.CreateDirectory(outputDir); } //Console.WriteLine("timePerResult"); TextWriter tw = new StreamWriter(outputDir + "timePerResult.txt"); foreach (double time in ana.timeTakenPerResult) { tw.WriteLine(time); } tw.Close(); tw = new StreamWriter(outputDir + "AcceptedTimePerSequenceNumber.txt"); foreach (int seq in ana.AcceptedTimePerSequenceNumber.Keys) { foreach (double time in ana.AcceptedTimePerSequenceNumber[seq]) { tw.WriteLine(time); } tw.WriteLine(""); } tw.Close(); tw = new StreamWriter(outputDir + "AcceptedTimePerTask.txt"); foreach (int taskid in ana.AcceptedTimePerTask.Keys) { foreach (double time in ana.AcceptedTimePerTask[taskid]) { tw.WriteLine(time); } tw.WriteLine(""); } tw.Close(); tw = new StreamWriter(outputDir + "TimePerSequenceNumber.txt"); foreach (int seq in ana.TimePerSequenceNumber.Keys) { foreach (double time in ana.TimePerSequenceNumber[seq]) { tw.WriteLine(time); } tw.WriteLine(""); } tw.Close(); tw = new StreamWriter(outputDir + "TimePerTask.txt"); foreach (int taskid in ana.TimePerTask.Keys) { foreach (double time in ana.TimePerTask[taskid]) { tw.WriteLine(time); } tw.WriteLine(""); } tw.Close(); //Console.WriteLine("resultArrivalTime"); TextWriter tw0 = new StreamWriter(outputDir + "resultArrivalTime.txt"); foreach (double time in ana.resultArrivalTimes) { tw0.WriteLine(time); } tw0.Close(); //Console.WriteLine("HITFinishTime"); TextWriter tw2 = new StreamWriter(outputDir + "HITFinishTime.txt"); foreach (double time in ana.finalHITEndTime.Values) { tw2.WriteLine(time); } tw2.Close(); //Console.WriteLine("SequenceNumberOfResultsPerTask"); TextWriter tw3 = new StreamWriter(outputDir + "SequenceNumberOfResultsPerTask.txt"); foreach (int seq in ana.SequenceNumberOfResultPerTask) { tw3.WriteLine(seq); } tw3.Close(); //Console.WriteLine("SequenceNumberOfResultsPerWorker"); TextWriter tw4 = new StreamWriter(outputDir + "SequenceNumberOfResultsPerWorker.txt"); foreach (int seq in ana.SequenceNumberOfResultPerWorker) { tw4.WriteLine(seq); } tw4.Close(); //Console.WriteLine("MoneyPaidPerResult"); TextWriter tw5 = new StreamWriter(outputDir + "MoneyPaidPerResult.txt"); foreach (int seq in ana.PricePerResult.Keys) { tw5.WriteLine(ana.PricePerResult[seq]); } tw5.Close(); TextWriter tw6 = new StreamWriter(outputDir + "acceptedTimePerResult.txt"); foreach (double time in ana.acceptedTimeTakenPerResult) { tw6.WriteLine(time); } tw6.Close(); ///////////////////// aggregation time histogram, out of scope //Console.WriteLine("resultArrivalTime", // TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MIN_RESULTS_TO_AGGREGATE, TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MAX_RESULTS_TO_AGGREGATE); //string filepath = String.Format(@"C:\research\MSRDVA\SatyamResearch\aggregationTimePerTask_MinNoResults_{0}_MaxNoResults_{1}.txt", // TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MIN_RESULTS_TO_AGGREGATE, TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MAX_RESULTS_TO_AGGREGATE); //TextWriter tw1 = new StreamWriter(filepath); //foreach (double time in ana.timeTakenTillAggregationPerTask.Values) //{ // tw1.WriteLine(time); //} //tw1.Close(); Console.WriteLine("Median Time For Accepted Task: {0}", ana.medianAcceptedTimeTakenPerResultInSeconds); Console.WriteLine("Median Time For All Task: {0}", ana.medianTimeTakenPerResultInSeconds); Console.WriteLine("Total Tasks: {0}", ana.noTasks); Console.WriteLine("Total Accepted Results: {0}", ana.totalAcceptedResults); Console.WriteLine("Total Results: {0}", ana.totalResults); Console.WriteLine("Avg Paid Results Per Task: {0}", ana.avgNoPaidResultsPerTask); return(ana); }
public static void Run([QueueTrigger("judge-result")] string myQueueItem, TraceWriter log) { bool logging = false; string[] fields = myQueueItem.Split('_'); string guid = fields[0]; int taskID = Convert.ToInt32(fields[1]); string resultID = fields[2]; if (logging) { log.Info($"Judge Result: {myQueueItem}"); } SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); List <SatyamResultsTableEntry> results = resultsDB.getEntriesID(resultID); resultsDB.close(); if (results.Count == 0) { return; } SatyamAggregatedResultsTableAccess aggDB = new SatyamAggregatedResultsTableAccess(); SatyamAggregatedResultsTableEntry aggResult = aggDB.getEntryByTaskID(taskID); aggDB.close(); if (aggResult == null) { return; } SatyamResultsTableEntry result = results[0]; SatyamResult res = JSonUtils.ConvertJSonToObject <SatyamResult>(result.ResultString); string taskTemplateType = result.JobTemplateType; string workerID = res.amazonInfo.WorkerID; DateTime doneTime = res.TaskEndTime; if (AcceptanceCriterionChecker.IsAcceptable(aggResult, result)) { resultsDB = new SatyamResultsTableAccess(); resultsDB.UpdateStatusByID(result.ID, ResultStatus.accepted); resultsDB.close(); WorkerStatisticsManagement.UpdateWorkerStatistics(workerID, taskTemplateType, true, doneTime); if (logging) { log.Info($"Accepted"); } } else { resultsDB = new SatyamResultsTableAccess(); resultsDB.UpdateStatusByID(result.ID, ResultStatus.rejected); resultsDB.close(); WorkerStatisticsManagement.UpdateWorkerStatistics(workerID, taskTemplateType, false, doneTime); if (logging) { log.Info($"Rejected"); } } }
public static void Run([TimerTrigger("0 */5 * * * *")] TimerInfo myTimer, TraceWriter log) { DateTime start = DateTime.Now; bool logging = false; if (logging) { log.Info($"Payment Dispatch executed at: {DateTime.Now}"); } SatyamDispatchStorageAccountAccess satyamQueue = new SatyamDispatchStorageAccountAccess(); //get IDs of all accepted and rejected results related to MTurk Dictionary <string, string> AssignmentIDToHITIDMap = new Dictionary <string, string>(); List <SatyamResultsTableEntry> acceptedEntries = new List <SatyamResultsTableEntry>(); foreach (string taskTemplate in TaskConstants.MTurkTaskTemplates) { SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); List <SatyamResultsTableEntry> entries = resultsDB.getAllEntriesByJobtemplateTypeAndStatus(taskTemplate, ResultStatus.accepted); resultsDB.close(); acceptedEntries.AddRange(entries); } List <SatyamResultsTableEntry> rejectedEntries = new List <SatyamResultsTableEntry>(); foreach (string taskTemplate in TaskConstants.MTurkTaskTemplates) { SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); List <SatyamResultsTableEntry> entries = resultsDB.getAllEntriesByJobtemplateTypeAndStatus(taskTemplate, ResultStatus.rejected); resultsDB.close(); rejectedEntries.AddRange(entries); } if (acceptedEntries.Count == 0 && rejectedEntries.Count == 0) //nothing to do { return; } // a single assignment may have multiple accepted results Dictionary <string, List <SatyamResultsTableEntry> > acceptedResultsByAssignmentID = new Dictionary <string, List <SatyamResultsTableEntry> >(); foreach (SatyamResultsTableEntry entry in acceptedEntries) { SatyamResult sr = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); string AssignmentID = sr.amazonInfo.AssignmentID; if (!AssignmentIDToHITIDMap.ContainsKey(AssignmentID)) { string HITID = sr.amazonInfo.HITID; AssignmentIDToHITIDMap.Add(AssignmentID, HITID); } if (!acceptedResultsByAssignmentID.ContainsKey(AssignmentID)) { acceptedResultsByAssignmentID.Add(AssignmentID, new List <SatyamResultsTableEntry>()); } acceptedResultsByAssignmentID[AssignmentID].Add(entry); } List <string> assignmentIDList = acceptedResultsByAssignmentID.Keys.ToList(); // a single assignment may have multiple rejected results Dictionary <string, List <SatyamResultsTableEntry> > rejectedResultsByAssignmentID = new Dictionary <string, List <SatyamResultsTableEntry> >(); foreach (SatyamResultsTableEntry entry in rejectedEntries) { SatyamResult sr = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); string AssignmentID = sr.amazonInfo.AssignmentID; if (!AssignmentIDToHITIDMap.ContainsKey(AssignmentID)) { string HITID = sr.amazonInfo.HITID; AssignmentIDToHITIDMap.Add(AssignmentID, HITID); } if (!rejectedResultsByAssignmentID.ContainsKey(AssignmentID)) { rejectedResultsByAssignmentID.Add(AssignmentID, new List <SatyamResultsTableEntry>()); if (!assignmentIDList.Contains(AssignmentID)) { assignmentIDList.Add(AssignmentID); } } rejectedResultsByAssignmentID[AssignmentID].Add(entry); } List <string> rejectedHITs = new List <string>(); List <string> acceptedHITs = new List <string>(); //for each assignment ID we check if it should be paid or not and then pay or reject them. //pay all accepted entries foreach (string assignmentID in assignmentIDList) { int noAccepted = 0; SatyamResult r = null; SatyamResultsTableEntry entry = null; if (acceptedResultsByAssignmentID.ContainsKey(assignmentID)) { noAccepted = acceptedResultsByAssignmentID[assignmentID].Count; entry = acceptedResultsByAssignmentID[assignmentID][0]; r = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); } int noRejected = 0; if (rejectedResultsByAssignmentID.ContainsKey(assignmentID)) { noRejected = rejectedResultsByAssignmentID[assignmentID].Count; if (r == null) { entry = rejectedResultsByAssignmentID[assignmentID][0]; r = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); } } double ratio = (double)noAccepted / ((double)noAccepted + (double)noRejected); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(r.TaskParametersString); string AmazonAccessKeyID = task.jobEntry.amazonHITInformation.AmazonAccessKeyID; string AmazonSecretAccessKeyID = task.jobEntry.amazonHITInformation.AmazonSecretAccessKeyID; string m = AmazonAccessKeyID + "_" + AmazonSecretAccessKeyID + "_" + assignmentID; SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); if (ratio >= AmazonMTurkPayments.assignement_acceptance_threshold) //this is acceptable { if (logging) { log.Info($"Dispatching Payment for {assignmentID}"); } string queueName = "payment"; satyamQueue.Enqueue(queueName, m); if (acceptedResultsByAssignmentID.ContainsKey(assignmentID)) { foreach (SatyamResultsTableEntry result in acceptedResultsByAssignmentID[assignmentID]) { resultsDB.UpdateStatusByID(result.ID, ResultStatus.accepted_Paid); } } if (rejectedResultsByAssignmentID.ContainsKey(assignmentID)) { foreach (SatyamResultsTableEntry result in rejectedResultsByAssignmentID[assignmentID]) { resultsDB.UpdateStatusByID(result.ID, ResultStatus.rejected_Paid); } } if (AssignmentIDToHITIDMap.ContainsKey(assignmentID)) { if (!acceptedHITs.Contains(AssignmentIDToHITIDMap[assignmentID])) { acceptedHITs.Add(AssignmentIDToHITIDMap[assignmentID]); SatyamAmazonHITTableAccess hitDB = new SatyamAmazonHITTableAccess(); hitDB.UpdateStatusByHITID(AssignmentIDToHITIDMap[assignmentID], HitStatus.accepted); hitDB.close(); } } } else { if (logging) { log.Info($"Dispatching NoPayment for {assignmentID}"); } string queueName = "nopayment"; satyamQueue.Enqueue(queueName, m); if (acceptedResultsByAssignmentID.ContainsKey(assignmentID)) { foreach (SatyamResultsTableEntry result in acceptedResultsByAssignmentID[assignmentID]) { resultsDB.UpdateStatusByID(result.ID, ResultStatus.accepted_NotPaid); } } if (rejectedResultsByAssignmentID.ContainsKey(assignmentID)) { foreach (SatyamResultsTableEntry result in rejectedResultsByAssignmentID[assignmentID]) { resultsDB.UpdateStatusByID(result.ID, ResultStatus.rejected_NotPaid); } } if (AssignmentIDToHITIDMap.ContainsKey(assignmentID)) { if (!rejectedHITs.Contains(AssignmentIDToHITIDMap[assignmentID])) { rejectedHITs.Add(AssignmentIDToHITIDMap[assignmentID]); SatyamAmazonHITTableAccess hitDB = new SatyamAmazonHITTableAccess(); hitDB.UpdateStatusByHITID(AssignmentIDToHITIDMap[assignmentID], HitStatus.rejected); hitDB.close(); } } } if ((DateTime.Now - start).TotalSeconds > 280) { break; } } }
public static void SaveResultImagesLocally(List <SatyamResultsTableEntry> entries, string directoryName) { if (!Directory.Exists(directoryName)) { Directory.CreateDirectory(directoryName); } directoryName = directoryName + "\\Raw"; if (!Directory.Exists(directoryName)) { Directory.CreateDirectory(directoryName); } for (int i = 0; i < entries.Count; i++) { SatyamResultsTableEntry entry = entries[i]; SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString); SatyamJob job = task.jobEntry; MultiObjectLocalizationAndLabelingResult res = JSonUtils.ConvertJSonToObject <MultiObjectLocalizationAndLabelingResult>(satyamResult.TaskResult); string ofilename = URIUtilities.filenameFromURI(task.SatyamURI); string[] fields = ofilename.Split('.'); string fileName = fields[0]; Image originalImage = ImageUtilities.getImageFromURI(task.SatyamURI); MultiObjectLocalizationAndLabelingSubmittedJob jobDefinition = JSonUtils.ConvertJSonToObject <MultiObjectLocalizationAndLabelingSubmittedJob>(job.JobParameters); Image imageWithBoundary = DrawingBoxesAndLinesOnImages.addLinesToImage(originalImage, jobDefinition.BoundaryLines, Color.Red, true); List <Rectangle> rectangles = new List <Rectangle>(); List <Color> colors = new List <Color>(); List <string> ids = new List <string>(); List <bool> dashed = new List <bool>(); for (int j = 0; j < res.objects.Count; j++) { MultiObjectLocalizationAndLabelingResultSingleEntry box = res.objects[j]; int x = box.boundingBox.tlx; int y = box.boundingBox.tly; int width = box.boundingBox.brx - box.boundingBox.tlx; int height = box.boundingBox.bry - box.boundingBox.tly; Rectangle r = new Rectangle(x, y, width, height); rectangles.Add(r); string category = box.Category; int colorIndex = jobDefinition.Categories.IndexOf(category); colors.Add(DrawingBoxesAndLinesOnImages.Colors[colorIndex]); string id = j + "-" + category; ids.Add(id); dashed.Add(false); } Image imageWithBoxesAndBoundary = DrawingBoxesAndLinesOnImages.addRectanglesToImage(imageWithBoundary, rectangles, colors, ids, dashed); fileName = fileName + "-Result"; if (satyamResult.amazonInfo.AssignmentID != "") { //fileName = fileName + "-" + satyamResult.amazonInfo.AssignmentID + "_" + entry.ID; fileName = fileName + "-" + entry.ID + "_" + satyamResult.PrevResultID; } ImageUtilities.saveImage(imageWithBoxesAndBoundary, directoryName, fileName); } }
public static bool IsAcceptable( SatyamAggregatedResultsTableEntry aggResultEntry, SatyamResultsTableEntry resultEntry, double ACCEPTANCE_NUMBER_OF_POLYGONS_THRESHOLD = TaskConstants.IMAGE_SEGMENTATION_MTURK_OBJECT_COVERAGE_THRESHOLD_FOR_PAYMENT, //the person must have made at least 80% of the boxes double POLYGON_IOU_THRESHOLD = TaskConstants.IMAGE_SEGMENTATION_MTURK_POLYGON_IOU_THRESHOLD_FOR_PAYMENT ) { //most boxes should be within limits //most categories should be right SatyamAggregatedResult satyamAggResult = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(aggResultEntry.ResultString); ImageSegmentationAggregatedResult_NoHoles aggresult = JSonUtils.ConvertJSonToObject <ImageSegmentationAggregatedResult_NoHoles>(satyamAggResult.AggregatedResultString); SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(resultEntry.ResultString); ImageSegmentationResult_NoHoles result = JSonUtils.ConvertJSonToObject <ImageSegmentationResult_NoHoles>(satyamResult.TaskResult); if (result == null) { return(false); } //first check if the number of boxes are within limit int boxLimit = (int)Math.Ceiling((double)aggresult.boxesAndCategories.objects.Count * ACCEPTANCE_NUMBER_OF_POLYGONS_THRESHOLD); if (result.objects.Count < boxLimit) { return(false); } //We fist do a bipartitte matching to find the best assocaition for the boxes List <List <GenericPolygon> > allPolygons = new List <List <GenericPolygon> >(); allPolygons.Add(new List <GenericPolygon>()); foreach (ImageSegmentationResultSingleEntry_NoHoles entry in result.objects) { allPolygons[0].Add(entry.polygon); } allPolygons.Add(new List <GenericPolygon>()); List <bool> tooSmallToIgnore = new List <bool>(); foreach (ImageSegmentationResultSingleEntry_NoHoles entry in aggresult.boxesAndCategories.objects) { allPolygons[1].Add(entry.polygon); } List <MultipartiteWeightedMatch> polygonAssociation = PolygonAssociation.computeGenericPolygonAssociations(allPolygons); //now find how many of the results match aggregated results int noAccepted = 0; foreach (MultipartiteWeightedMatch match in polygonAssociation) { if (match.elementList.ContainsKey(1)) // this contains an aggregated box { if (match.elementList.ContainsKey(0)) // a result box has been associated { GenericPolygon aggregatedGenericPolygon = allPolygons[1][match.elementList[1]]; GenericPolygon resultGenericPolygon = allPolygons[0][match.elementList[0]]; //double IoU = GenericPolygon.computeIntersectionOverUnion(aggregatedGenericPolygon, resultGenericPolygon); double IoU = 1; if (IoU >= POLYGON_IOU_THRESHOLD) { //now check category if (result.objects[match.elementList[0]].Category == aggresult.boxesAndCategories.objects[match.elementList[1]].Category) { //both category and bounding box tests have passed noAccepted++; } } } } } if (noAccepted >= boxLimit) { return(true); } return(false); }
public static bool IsAcceptable( SatyamAggregatedResultsTableEntry aggResultEntry, SatyamResultsTableEntry resultEntry, double APPROVAL_RATIO_OF_POLYGONS_THRESHOLD = TaskConstants.IMAGE_SEGMENTATION_MTURK_OBJECT_COVERAGE_THRESHOLD_FOR_PAYMENT, //the person must have made at least 80% of the polygon double POLYGON_IOU_THRESHOLD = TaskConstants.IMAGE_SEGMENTATION_MTURK_POLYGON_IOU_THRESHOLD_FOR_PAYMENT ) { //most boxes should be within limits //most categories should be right SatyamAggregatedResult satyamAggResult = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(aggResultEntry.ResultString); ImageSegmentationAggregatedResult aggresult = JSonUtils.ConvertJSonToObject <ImageSegmentationAggregatedResult>(satyamAggResult.AggregatedResultString); SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(resultEntry.ResultString); ImageSegmentationResult result = JSonUtils.ConvertJSonToObject <ImageSegmentationResult>(satyamResult.TaskResult); if (result == null) { return(false); } //first check if the number of boxes are within limit // the objects are dummy polygons just for counting int boxLimit = (int)Math.Ceiling((double)aggresult.boxesAndCategories.objects.Count * APPROVAL_RATIO_OF_POLYGONS_THRESHOLD); if (result.objects.Count < boxLimit) { return(false); } //now find how many of the results match aggregated results int noAccepted = 0; byte[] resPNG = ImageSegmentationResult.PolygonResult2Bitmap(result); int width = -1; int height = -1; byte[] aggPNG = ImageUtilities.readPNGRawDataFromURL(aggresult.metaData.PNG_URL, out width, out height); if (resPNG.Length != aggPNG.Length) { Console.WriteLine("res and agg res are different size"); return(false); } SortedDictionary <int, SortedDictionary <int, int> > overlapping = new SortedDictionary <int, SortedDictionary <int, int> >(); SortedDictionary <int, int> resObjectArea = new SortedDictionary <int, int>(); SortedDictionary <int, int> aggObjectArea = new SortedDictionary <int, int>(); for (int i = 0; i < width; i++) { for (int j = 0; j < height; j++) { int idx = j * width + i; int resObjID = resPNG[idx]; int aggObjID = aggPNG[idx]; if (!overlapping.ContainsKey(resObjID)) { overlapping.Add(resObjID, new SortedDictionary <int, int>()); } if (!overlapping[resObjID].ContainsKey(aggObjID)) { overlapping[resObjID].Add(aggObjID, 0); } overlapping[resObjID][aggObjID]++; if (!resObjectArea.ContainsKey(resObjID)) { resObjectArea.Add(resObjID, 0); } resObjectArea[resObjID]++; if (!aggObjectArea.ContainsKey(aggObjID)) { aggObjectArea.Add(aggObjID, 0); } aggObjectArea[aggObjID]++; } } foreach (int id in resObjectArea.Keys) { if (id == 0) { continue; } int maxOverlap = -1; int maxOverlapAggObjID = -1; SortedDictionary <int, int> overlapArea = overlapping[id]; foreach (int aggid in overlapArea.Keys) { if (aggid == 0) { continue; } if (overlapArea[aggid] > maxOverlap) { maxOverlap = overlapArea[aggid]; maxOverlapAggObjID = aggid; } } if (maxOverlapAggObjID == -1) { continue; } double iou = (double)maxOverlap / (double)(resObjectArea[id] + aggObjectArea[maxOverlapAggObjID] - maxOverlap); if (iou >= POLYGON_IOU_THRESHOLD) { noAccepted++; ////now check category //if (result.objects[match.elementList[0]].Category == aggresult.boxesAndCategories.objects[match.elementList[1]].Category) //{ // //both category and bounding box tests have passed //} } } if (noAccepted >= boxLimit) { return(true); } return(false); }
public static double ACCEPTANCE_NUMBER_OF_BOXES_THRESHOLD = TaskConstants.MULTI_OBJECT_LOCALIZATION_AND_LABLING_MTURK_OBJECT_COVERAGE_THRESHOLD_FOR_PAYMENT; //the person must have made at least 80% of the boxes public static bool IsAcceptable(SatyamAggregatedResultsTableEntry aggResultEntry, SatyamResultsTableEntry resultEntry, double DeviationPixelThreshold = TaskConstants.MULTI_OBJECT_LOCALIZATION_AND_LABLING_MTURK_DEVIATION_THRESHOLD_FOR_PAYMENT) { //most boxes should be within limits //most categories should be right SatyamAggregatedResult satyamAggResult = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(aggResultEntry.ResultString); MultiObjectLocalizationAndLabelingAggregatedResult aggresult = JSonUtils.ConvertJSonToObject <MultiObjectLocalizationAndLabelingAggregatedResult>(satyamAggResult.AggregatedResultString); SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(resultEntry.ResultString); MultiObjectLocalizationAndLabelingResult result = JSonUtils.ConvertJSonToObject <MultiObjectLocalizationAndLabelingResult>(satyamResult.TaskResult); if (result == null) { return(false); } //first check if the number of boxes are within limit int boxLimit = (int)Math.Ceiling((double)aggresult.boxesAndCategories.objects.Count * ACCEPTANCE_NUMBER_OF_BOXES_THRESHOLD); if (result.objects.Count < boxLimit) { return(false); } double minboxdimension_threshold_x = MIN_BOX_DIMENSION_FOR_CONSIDERATION * result.displayScaleReductionX; double minboxdimension_threshold_y = MIN_BOX_DIMENSION_FOR_CONSIDERATION * result.displayScaleReductionY; //We fist do a bipartitte matching to find the best assocaition for the boxes List <List <BoundingBox> > allboxes = new List <List <BoundingBox> >(); allboxes.Add(new List <BoundingBox>()); foreach (MultiObjectLocalizationAndLabelingResultSingleEntry entry in result.objects) { allboxes[0].Add(entry.boundingBox); } allboxes.Add(new List <BoundingBox>()); List <bool> tooSmallToIgnore = new List <bool>(); foreach (MultiObjectLocalizationAndLabelingResultSingleEntry entry in aggresult.boxesAndCategories.objects) { allboxes[1].Add(entry.boundingBox); if (entry.boundingBox.getWidth() < minboxdimension_threshold_x && entry.boundingBox.getHeight() < minboxdimension_threshold_y) { tooSmallToIgnore.Add(true); } else { tooSmallToIgnore.Add(false); } } List <MultipartiteWeightedMatch> boxAssociation = BoundingBoxAssociation.computeBoundingBoxAssociations(allboxes); //now find how many of the results match aggregated results int noAccepted = 0; double deviation_threshold_x = DeviationPixelThreshold * result.displayScaleReductionX; double deviation_threshold_y = DeviationPixelThreshold * result.displayScaleReductionY; int noIgnorable = 0; foreach (MultipartiteWeightedMatch match in boxAssociation) { if (match.elementList.ContainsKey(1)) // this contains an aggregated box { if (match.elementList.ContainsKey(0)) // a result box has been associated { BoundingBox aggregatedBoundingBox = allboxes[1][match.elementList[1]]; BoundingBox resultBoundingBox = allboxes[0][match.elementList[0]]; //double deviation = aggregatedBoundingBox.ComputeMaxDeviationMetric(resultBoundingBox); double deviation = aggregatedBoundingBox.ComputeNormalizedMaxDeviationMetric(resultBoundingBox, deviation_threshold_x, deviation_threshold_y); if (deviation <= 1) //deviation test passed { //now check category if (result.objects[match.elementList[0]].Category == aggresult.boxesAndCategories.objects[match.elementList[1]].Category) { //both category and bounding box tests have passed noAccepted++; } } else { if (tooSmallToIgnore[match.elementList[1]]) { noIgnorable++; } } } } } if (noAccepted >= boxLimit - noIgnorable) { return(true); } return(false); }
//go through all unpaid results //check if aggregated results has an entry for this //check if it satisfies acceptance criterion //pay if passes else fail! public static void MakePayments() { //get IDs of all accepted and rejected results related to MTurk SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); Dictionary <string, string> AssignmentIDToHITIDMap = new Dictionary <string, string>(); List <SatyamResultsTableEntry> acceptedEntries = new List <SatyamResultsTableEntry>(); foreach (string taskTemplate in TaskConstants.MTurkTaskTemplates) { List <SatyamResultsTableEntry> entries = resultsDB.getAllEntriesByJobtemplateTypeAndStatus(taskTemplate, ResultStatus.accepted); acceptedEntries.AddRange(entries); } List <SatyamResultsTableEntry> rejectedEntries = new List <SatyamResultsTableEntry>(); foreach (string taskTemplate in TaskConstants.MTurkTaskTemplates) { List <SatyamResultsTableEntry> entries = resultsDB.getAllEntriesByJobtemplateTypeAndStatus(taskTemplate, ResultStatus.rejected); rejectedEntries.AddRange(entries); } resultsDB.close(); if (acceptedEntries.Count == 0 && rejectedEntries.Count == 0) //nothing to do { return; } // a single assignment may have multiple accepted results Dictionary <string, List <SatyamResultsTableEntry> > acceptedResultsByAssignmentID = new Dictionary <string, List <SatyamResultsTableEntry> >(); foreach (SatyamResultsTableEntry entry in acceptedEntries) { SatyamResult sr = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); string AssignmentID = sr.amazonInfo.AssignmentID; if (!AssignmentIDToHITIDMap.ContainsKey(AssignmentID)) { string HITID = sr.amazonInfo.HITID; AssignmentIDToHITIDMap.Add(AssignmentID, HITID); } if (!acceptedResultsByAssignmentID.ContainsKey(AssignmentID)) { acceptedResultsByAssignmentID.Add(AssignmentID, new List <SatyamResultsTableEntry>()); } acceptedResultsByAssignmentID[AssignmentID].Add(entry); } List <string> assignmentIDList = acceptedResultsByAssignmentID.Keys.ToList(); // a single assignment may have multiple rejected results Dictionary <string, List <SatyamResultsTableEntry> > rejectedResultsByAssignmentID = new Dictionary <string, List <SatyamResultsTableEntry> >(); foreach (SatyamResultsTableEntry entry in rejectedEntries) { SatyamResult sr = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); string AssignmentID = sr.amazonInfo.AssignmentID; if (!AssignmentIDToHITIDMap.ContainsKey(AssignmentID)) { string HITID = sr.amazonInfo.HITID; AssignmentIDToHITIDMap.Add(AssignmentID, HITID); } if (!rejectedResultsByAssignmentID.ContainsKey(AssignmentID)) { rejectedResultsByAssignmentID.Add(AssignmentID, new List <SatyamResultsTableEntry>()); if (!assignmentIDList.Contains(AssignmentID)) { assignmentIDList.Add(AssignmentID); } } rejectedResultsByAssignmentID[AssignmentID].Add(entry); } List <string> rejectedHITs = new List <string>(); List <string> acceptedHITs = new List <string>(); //for each assignment ID we check if it should be paid or not and then pay or reject them. //pay all accepted entries AmazonMTurkHIT hit = new AmazonMTurkHIT(); foreach (string assignmentID in assignmentIDList) { int noAccepted = 0; SatyamResult r = null; SatyamResultsTableEntry entry = null; if (acceptedResultsByAssignmentID.ContainsKey(assignmentID)) { noAccepted = acceptedResultsByAssignmentID[assignmentID].Count; entry = acceptedResultsByAssignmentID[assignmentID][0]; r = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); } int noRejected = 0; if (rejectedResultsByAssignmentID.ContainsKey(assignmentID)) { noRejected = rejectedResultsByAssignmentID[assignmentID].Count; if (r == null) { entry = rejectedResultsByAssignmentID[assignmentID][0]; r = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); } } double ratio = (double)noAccepted / ((double)noAccepted + (double)noRejected); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(r.TaskParametersString); string AmazonAccessKeyID = task.jobEntry.amazonHITInformation.AmazonAccessKeyID; string AmazonSecretAccessKeyID = task.jobEntry.amazonHITInformation.AmazonSecretAccessKeyID; hit.setAccount(AmazonAccessKeyID, AmazonSecretAccessKeyID, false); resultsDB = new SatyamResultsTableAccess(); if (ratio >= assignement_acceptance_threshold) //this is acceptable { hit.ApproveAssignment(assignmentID, "Great Job! Your work was within acceptable parameters!"); if (acceptedResultsByAssignmentID.ContainsKey(assignmentID)) { foreach (SatyamResultsTableEntry result in acceptedResultsByAssignmentID[assignmentID]) { resultsDB.UpdateStatusByID(result.ID, ResultStatus.accepted_Paid); } } if (rejectedResultsByAssignmentID.ContainsKey(assignmentID)) { foreach (SatyamResultsTableEntry result in rejectedResultsByAssignmentID[assignmentID]) { resultsDB.UpdateStatusByID(result.ID, ResultStatus.rejected_Paid); } } if (AssignmentIDToHITIDMap.ContainsKey(assignmentID)) { if (!acceptedHITs.Contains(AssignmentIDToHITIDMap[assignmentID])) { acceptedHITs.Add(AssignmentIDToHITIDMap[assignmentID]); } } } else { hit.RejectAssignment(assignmentID, "Sorry! Your work was not within acceptable parameters!"); if (acceptedResultsByAssignmentID.ContainsKey(assignmentID)) { foreach (SatyamResultsTableEntry result in acceptedResultsByAssignmentID[assignmentID]) { resultsDB.UpdateStatusByID(result.ID, ResultStatus.accepted_NotPaid); } } if (rejectedResultsByAssignmentID.ContainsKey(assignmentID)) { foreach (SatyamResultsTableEntry result in rejectedResultsByAssignmentID[assignmentID]) { resultsDB.UpdateStatusByID(result.ID, ResultStatus.rejected_NotPaid); } } if (AssignmentIDToHITIDMap.ContainsKey(assignmentID)) { if (!rejectedHITs.Contains(AssignmentIDToHITIDMap[assignmentID])) { rejectedHITs.Add(AssignmentIDToHITIDMap[assignmentID]); } } } resultsDB.close(); } //update the status of all finished HITs in the hittable SatyamAmazonHITTableAccess hitDB = new SatyamAmazonHITTableAccess(); foreach (string HITID in acceptedHITs) { hitDB.UpdateStatusByHITID(HITID, HitStatus.accepted); } foreach (string HITID in rejectedHITs) { hitDB.UpdateStatusByHITID(HITID, HitStatus.rejected); } hitDB.close(); }
public static void ValidateSameTaskSameWorkerMultipleResultsEffect(string jobGUID, string confusingImageListFilePath = null) { SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); List <SatyamResultsTableEntry> entries = resultsDB.getEntriesByGUIDOrderByID(jobGUID); resultsDB.close(); Dictionary <int, int> noResultsPerTask = new Dictionary <int, int>(); Dictionary <int, int> noAcceptedResultsPerTask = new Dictionary <int, int>(); Dictionary <int, int> noPaidResultsPerTask = new Dictionary <int, int>(); Dictionary <int, double> moneyPaidPerTask = new Dictionary <int, double>(); SortedDictionary <int, Dictionary <string, List <string> > > ResultsPerWorkerPerTask = new SortedDictionary <int, Dictionary <string, List <string> > >(); SortedDictionary <int, Dictionary <string, int> > noDifferentDuplicateResultsPerWorkerPerTask = new SortedDictionary <int, Dictionary <string, int> >(); Dictionary <int, string> taskSatyamUri = new Dictionary <int, string>(); Dictionary <int, double> timeTakenTillAggregationPerTask = new Dictionary <int, double>(); Dictionary <int, DateTime> finalTaskEndTime = new Dictionary <int, DateTime>(); Dictionary <string, double> finalHITEndTime = new Dictionary <string, double>(); int totalResults = 0; int noTasksWithDuplicateResultsFromSameWorker = 0; int noTasksWithMixedResultsFromSameWorker = 0; int noTaskWithDuplicateResultsAsMajority = 0; int noTaskWhoseDuplicateResultsChangedAggregation = 0; int noTaskWhoseDuplicateResultsChangedAggregationIncorrectly = 0; int noCorrectDecisionsAmongDuplicates = 0; //noCorrectDuplicateResultsOfSameWorkerSameTask = 0; int noTasksWithCorrectDuplicateResultsOfSameWorkerSameTask = 0; int noWorkerSwitchedToCorrect = 0; int noWorkerSwitchedToCorrectAndMaintained = 0; int noTasksWithWorkerSwitchedToCorrect = 0; int noTasksWithWorkerSwitchedToCorrectAndMaintained = 0; List <double> timeTakenPerResult = new List <double>(); List <double> acceptedTimeTakenPerResult = new List <double>(); List <int> SequenceNumberOfResultPerTask = new List <int>(); List <int> SequenceNumberOfResultPerWorker = new List <int>(); string JobGUID = entries[0].JobGUID; Dictionary <string, int> noJobsPerWorker = new Dictionary <string, int>(); Dictionary <string, int> noAcceptedJobsPerWorker = new Dictionary <string, int>(); Dictionary <string, int> noPaidJobsPerWorker = new Dictionary <string, int>(); List <double> resultArrivalTimes = new List <double>(); totalResults = entries.Count; ///imagenet only Dictionary <string, ConfusingReason> imageBlackListReason = new Dictionary <string, ConfusingReason>(); if (confusingImageListFilePath != null) { imageBlackListReason = getConfusingImageList(confusingImageListFilePath); } for (int i = 0; i < entries.Count; i++) { SatyamResultsTableEntry entry = entries[i]; SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString); SatyamJob job = task.jobEntry; if (IsBlackListed(task.SatyamURI, imageBlackListReason)) { continue; } int taskEntryID = entry.SatyamTaskTableEntryID; if (!noResultsPerTask.ContainsKey(taskEntryID)) { noResultsPerTask.Add(taskEntryID, 0); noAcceptedResultsPerTask.Add(taskEntryID, 0); noPaidResultsPerTask.Add(taskEntryID, 0); moneyPaidPerTask.Add(taskEntryID, 0); finalTaskEndTime.Add(taskEntryID, entry.SubmitTime); ResultsPerWorkerPerTask.Add(taskEntryID, new Dictionary <string, List <string> >()); noDifferentDuplicateResultsPerWorkerPerTask.Add(taskEntryID, new Dictionary <string, int>()); taskSatyamUri.Add(taskEntryID, task.SatyamURI); } if (!ResultsPerWorkerPerTask[taskEntryID].ContainsKey(satyamResult.amazonInfo.WorkerID)) { ResultsPerWorkerPerTask[taskEntryID].Add(satyamResult.amazonInfo.WorkerID, new List <string>()); noDifferentDuplicateResultsPerWorkerPerTask[taskEntryID].Add(satyamResult.amazonInfo.WorkerID, 0); } if (!ResultsPerWorkerPerTask[taskEntryID][satyamResult.amazonInfo.WorkerID].Contains(satyamResult.TaskResult)) { noDifferentDuplicateResultsPerWorkerPerTask[taskEntryID][satyamResult.amazonInfo.WorkerID]++; } ResultsPerWorkerPerTask[taskEntryID][satyamResult.amazonInfo.WorkerID].Add(satyamResult.TaskResult); } ///////////////////////////////////// Per Task Analysis ////////////////////////////////////// SortedDictionary <int, int> resultsPerTaskHistogram = new SortedDictionary <int, int>(); SortedDictionary <int, int> resultsAcceptedPerTaskHistogram = new SortedDictionary <int, int>(); SortedDictionary <int, int> resultsPaidPerTaskHistogram = new SortedDictionary <int, int>(); SortedDictionary <int, int> moneyPaidPerTaskHistogram = new SortedDictionary <int, int>(); //cents SortedDictionary <int, int> ResultsPerWorkerPerTaskHistogram = new SortedDictionary <int, int>(); List <int> taskIDs = noResultsPerTask.Keys.ToList(); foreach (int taskID in taskIDs) { if (ResultsPerWorkerPerTask[taskID].Count != noResultsPerTask[taskID]) { //has multiple results from same turker noTasksWithDuplicateResultsFromSameWorker++; // the aggregation result List <SingleObjectLabelingResult> allResultsPerTask = new List <SingleObjectLabelingResult>(); SingleObjectLabelingAggregatedResult aggregatedResultPerTask = new SingleObjectLabelingAggregatedResult(); // the aggregation if without duplicate results List <SingleObjectLabelingResult> OnlyFirstResultOfEachTurkerPerTask = new List <SingleObjectLabelingResult>(); SingleObjectLabelingAggregatedResult aggregatedOnlyFirstResultOfTurkerPerTask = new SingleObjectLabelingAggregatedResult(); foreach (List <string> ResultsStringsPerWorkerPerTask in ResultsPerWorkerPerTask[taskID].Values) { foreach (string resultString in ResultsStringsPerWorkerPerTask) { allResultsPerTask.Add(JSonUtils.ConvertJSonToObject <SingleObjectLabelingResult>(resultString)); } OnlyFirstResultOfEachTurkerPerTask.Add(JSonUtils.ConvertJSonToObject <SingleObjectLabelingResult>(ResultsStringsPerWorkerPerTask[0])); } aggregatedResultPerTask = SingleObjectLabelingAggregator.getAggregatedResult(allResultsPerTask); aggregatedOnlyFirstResultOfTurkerPerTask = SingleObjectLabelingAggregator.getAggregatedResult(OnlyFirstResultOfEachTurkerPerTask); if (aggregatedResultPerTask != null) { if (aggregatedOnlyFirstResultOfTurkerPerTask == null || aggregatedResultPerTask.Category != aggregatedOnlyFirstResultOfTurkerPerTask.Category) { noTaskWhoseDuplicateResultsChangedAggregation++; if (AggregatedResultEqualsGroundTruth(taskSatyamUri[taskID], JSonUtils.ConvertObjectToJSon(aggregatedResultPerTask))) { noTaskWhoseDuplicateResultsChangedAggregationIncorrectly++; } } } bool hasMixedResults = false; bool hasCorrectResult = false; bool atLeastOneShiftedFromIncorrectToCorrect = false; bool atLeastOneShiftedFromIncorrectToCorrectAndMaintained = false; foreach (List <string> ResultsStringsPerWorkerPerTask in ResultsPerWorkerPerTask[taskID].Values) { if (!ResultsPerWorkerPerTaskHistogram.ContainsKey(ResultsStringsPerWorkerPerTask.Count)) { ResultsPerWorkerPerTaskHistogram.Add(ResultsStringsPerWorkerPerTask.Count, 0); } ResultsPerWorkerPerTaskHistogram[ResultsStringsPerWorkerPerTask.Count]++; double superMajority = 0.6; // tunable if (((double)ResultsStringsPerWorkerPerTask.Count + 1) / ((double)noResultsPerTask[taskID] + 2) > superMajority) { noTaskWithDuplicateResultsAsMajority++; } ///////////// imagenet only ////////////////// if (ResultsStringsPerWorkerPerTask.Distinct().Count() > 1) { // multiple choices given hasMixedResults = true; bool incorrect = false; bool switchToCorrect = false; foreach (string resultString in ResultsStringsPerWorkerPerTask) { if (SingleObjectLabelingResultEqualsGroundTruth(taskSatyamUri[taskID], resultString)) { hasCorrectResult = true; noCorrectDecisionsAmongDuplicates++; if (incorrect) { switchToCorrect = true; } incorrect = false; } else { incorrect = true; } } if (switchToCorrect) { // been incorrect and swithed to correct noWorkerSwitchedToCorrect++; atLeastOneShiftedFromIncorrectToCorrect = true; if (!incorrect) { //switch to correct and maintain till the end noWorkerSwitchedToCorrectAndMaintained++; atLeastOneShiftedFromIncorrectToCorrectAndMaintained = true; } } } //////////////////////////////////////////////// } if (hasMixedResults) { noTasksWithMixedResultsFromSameWorker++; } if (hasCorrectResult) { noTasksWithCorrectDuplicateResultsOfSameWorkerSameTask++; } if (atLeastOneShiftedFromIncorrectToCorrect) { noTasksWithWorkerSwitchedToCorrect++; } if (atLeastOneShiftedFromIncorrectToCorrectAndMaintained) { noTasksWithWorkerSwitchedToCorrectAndMaintained++; } } } Console.WriteLine("DuplicateResultsHistogram"); foreach (int no in ResultsPerWorkerPerTaskHistogram.Keys) { Console.WriteLine("{0}, {1}", no, ResultsPerWorkerPerTaskHistogram[no]); } Console.WriteLine( //"{0} images (in total {1}({2} of which are correct)) has duplicate results from same turker, \n" + //"\t{3} images have >=1 workers making more than majority number of results.\n" + //"\t{11} images aggregation is changed ({12} of which incorrectly) by duplicate results from same turker.\n" + //"\t{4} images(In total {5} times) with duplicate results include mixed(>= 2) decisions from same worker.\n" + "\t\t{6} images has correct decision among duplicate decisions.\n" + "\t\t{7} images({8} times) a worker has switched from incorrect to correct choice\n" + "\t\t\t{9} images({10} times) of which maintained the correct choice till the last time of their job on that image.", //noTasksWithDuplicateResultsFromSameWorker, noDuplicateResultsFromSameWorker, noCorrectDecisionsAmongDuplicates, //noTaskWithDuplicateResultsAsMajority, //noTasksWithMixedResultsFromSameWorker, noDecisionChangeAmongDuplicateResultsOfSameWorkerSameTask, //noTasksWithCorrectDuplicateResultsOfSameWorkerSameTask, noTasksWithWorkerSwitchedToCorrect, noWorkerSwitchedToCorrect, noTasksWithWorkerSwitchedToCorrectAndMaintained, noWorkerSwitchedToCorrectAndMaintained, noTaskWhoseDuplicateResultsChangedAggregation, noTaskWhoseDuplicateResultsChangedAggregationIncorrectly ); }
public static void SaveResultImagesLocally(List <SatyamResultsTableEntry> entries, string directoryName) { if (!Directory.Exists(directoryName)) { Directory.CreateDirectory(directoryName); } directoryName = directoryName + "\\Raw"; if (!Directory.Exists(directoryName)) { Directory.CreateDirectory(directoryName); } // sort by task id SortedDictionary <int, List <SatyamResultsTableEntry> > taskResults = new SortedDictionary <int, List <SatyamResultsTableEntry> >(); for (int i = 0; i < entries.Count; i++) { if (!taskResults.ContainsKey(entries[i].SatyamTaskTableEntryID)) { taskResults.Add(entries[i].SatyamTaskTableEntryID, new List <SatyamResultsTableEntry>()); } taskResults[entries[i].SatyamTaskTableEntryID].Add(entries[i]); } foreach (int taskID in taskResults.Keys) { for (int i = 0; i < taskResults[taskID].Count; i++) { SatyamResultsTableEntry entry = taskResults[taskID][i]; SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString); SatyamJob job = task.jobEntry; ImageSegmentationResult res = JSonUtils.ConvertJSonToObject <ImageSegmentationResult>(satyamResult.TaskResult); if (res == null) { continue; } string fileName = URIUtilities.filenameFromURINoExtension(task.SatyamURI); fileName = fileName + "-Result"; if (satyamResult.amazonInfo.AssignmentID != "") { fileName = fileName + "-" + satyamResult.amazonInfo.AssignmentID; } fileName += "-" + entry.ID; if (File.Exists(directoryName + "\\" + fileName + ".jpg")) { continue; } Console.WriteLine("Saving Turker Result {0}", fileName); Image originalImage = ImageUtilities.getImageFromURI(task.SatyamURI); Image ResultImage = DrawResultsOnImage(res, originalImage); byte[] png = ImageSegmentationResult.PolygonResult2Bitmap(res); if (ResultImage == null) { continue; } ImageUtilities.saveImage(ResultImage, directoryName, fileName); ImageUtilities.savePNGRawData(directoryName + "\\" + fileName + "_bitmap.jpg", originalImage.Width, originalImage.Height, png); } } }