public static void AggregateWithParameterAndValidateSatyamVideoClassificationResultByGUID(string guid, int MinResults = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MIN_RESULTS_TO_AGGREGATE, int MaxResults = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MAX_RESULTS_TO_AGGREGATE, double MajorityThreshold = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MAJORITY_THRESHOLD, bool prepareDataForTraining = false, string outputDirectory = null, string confusingVideoListFilePath = null) { string configString = "Min_" + MinResults + "_Max_" + MaxResults + "_Thresh_" + MajorityThreshold; Console.WriteLine("Aggregating for param set " + configString); int noTerminatedTasks = 0; SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); List <SatyamResultsTableEntry> entries = resultsDB.getEntriesByGUIDOrderByID(guid); resultsDB.close(); SortedDictionary <DateTime, List <SatyamResultsTableEntry> > entriesBySubmitTime = SatyamResultValidationToolKit.SortResultsBySubmitTime(entries); int noTotalConverged = 0; int noCorrect = 0; Dictionary <int, List <SatyamResultsTableEntry> > ResultsPerTask = new Dictionary <int, List <SatyamResultsTableEntry> >(); List <int> aggregatedTasks = new List <int>(); SortedDictionary <string, Dictionary <string, int> > confusionMatrix_res_groundtruth = new SortedDictionary <string, Dictionary <string, int> >(); SortedDictionary <string, Dictionary <string, int> > confusionMatrix_groundtruth_res = new SortedDictionary <string, Dictionary <string, int> >(); List <SatyamAggregatedResultsTableEntry> aggEntries = new List <SatyamAggregatedResultsTableEntry>(); Dictionary <int, int> noResultsNeededForAggregation = SatyamResultsAnalysis.getNoResultsNeededForAggregationFromLog(configString, guid); Dictionary <int, int> noResultsNeededForAggregation_new = new Dictionary <int, int>(); foreach (DateTime t in entriesBySubmitTime.Keys) { //Console.WriteLine("Processing Results of time: {0}", t); foreach (SatyamResultsTableEntry entry in entriesBySubmitTime[t]) { SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString); SatyamJob job = task.jobEntry; string fileName = URIUtilities.filenameFromURI(task.SatyamURI); string VideoCategoryGroundTruth = getVideoCategoryFromFileName(fileName); int taskEntryID = entry.SatyamTaskTableEntryID; if (aggregatedTasks.Contains(taskEntryID)) { continue; } if (!ResultsPerTask.ContainsKey(taskEntryID)) { ResultsPerTask.Add(taskEntryID, new List <SatyamResultsTableEntry>()); } ResultsPerTask[taskEntryID].Add(entry); // check log if enough results are collected if (noResultsNeededForAggregation != null && noResultsNeededForAggregation.ContainsKey(taskEntryID) && ResultsPerTask[taskEntryID].Count < noResultsNeededForAggregation[taskEntryID]) { continue; } //SingleObjectLabelingAggregatedResult aggResult = SingleObjectLabelingAggregator.getAggregatedResult(ResultsPerTask[taskEntryID], MinResults, MaxResult, MajorityThreshold); string aggResultString = SingleObjectLabelingAggregator.GetAggregatedResultString(ResultsPerTask[taskEntryID], MinResults, MaxResults, MajorityThreshold); if (aggResultString == null) { continue; } SatyamAggregatedResultsTableEntry aggEntry = new SatyamAggregatedResultsTableEntry(); aggEntry.JobGUID = ResultsPerTask[taskEntryID][0].JobGUID; aggEntry.JobTemplateType = ResultsPerTask[taskEntryID][0].JobTemplateType; aggEntry.SatyamTaskTableEntryID = taskEntryID; aggEntry.UserID = ResultsPerTask[taskEntryID][0].UserID; aggEntry.ResultString = aggResultString; /// aggregation happen // record logs if (noResultsNeededForAggregation == null || !noResultsNeededForAggregation.ContainsKey(taskEntryID)) { noResultsNeededForAggregation_new.Add(taskEntryID, ResultsPerTask[taskEntryID].Count); } aggEntries.Add(aggEntry); noTotalConverged++; if (ResultsPerTask[taskEntryID].Count >= MaxResults) { noTerminatedTasks++; } aggregatedTasks.Add(taskEntryID); } } SatyamResultsAnalysis.RecordAggregationLog(noResultsNeededForAggregation_new, configString, guid); string outputmatFile = DirectoryConstants.defaultTempDirectory + guid + "\\" + configString + "_mat.txt"; EvaluateAndPrintConfusionMatrixOfAggregatedResultEntries(aggEntries, outputmatFile, out noCorrect, prepareDataForTraining, outputDirectory); Console.WriteLine("noTotalConverged {0}", noTotalConverged); Console.WriteLine("noTerminatedTasks {0}", noTerminatedTasks); Console.WriteLine("Result: {0}/{1}, precision: {2}", noCorrect, noTotalConverged, (double)noCorrect / noTotalConverged); /// local file string outputString = String.Format("{0} {1} {2} {3} {4} {5}\n", configString, noCorrect, noTotalConverged, (double)noCorrect / noTotalConverged, noTerminatedTasks, ResultsPerTask.Count - noTotalConverged); string outputfile = DirectoryConstants.defaultTempDirectory + guid + "\\resultSummary.txt"; File.AppendAllText(outputfile, outputString); //for (double prob = 0; prob < 1;prob +=0.2) //{ // SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entries, guid, configString, anotherChanceProbablity: prob); //} //for (double ratio = 0; ratio < 1; ratio += 0.2) //{ // SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guid, configString, approvalRatioThreshold: ratio); //} SatyamResultsAnalysis.AggregationAnalysis(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guid, configString); }
public static void AggregateWithParameter(string guid, int MinResults = TaskConstants.TRACKLET_LABELING_MTURK_MIN_RESULTS_TO_AGGREGATE, int MaxResults = TaskConstants.TRACKLET_LABELING_MTURK_MAX_RESULTS_TO_AGGREGATE, double boxToleranceThreshold = TaskConstants.TRACKLET_LABELING_BOX_DEVIATION_THRESHOLD, double ObjectCoverageApprovalThresholdPerVideo = TaskConstants.TRACKLET_LABELING_APPROVALRATIO_PER_VIDEO, double BoxCoverageApprovalThresholdPerTrack = TaskConstants.TRACKLET_LABELING_APPROVALRATIO_PER_TRACK, int consensusNumber = TaskConstants.TRACKLET_LABELING_MIN_RESULTS_FOR_CONSENSUS, double minTubeletIoUSimilarityThreshold = TaskConstants.TRACKLET_LABELING_MIN_TUBELET_SIMILARITY_THRESHOLD, double attributeMajority = TaskConstants.TRACKLET_LABELING_MTURK_ATTRIBUTE_MAJORITY_THRESHOLD, bool allFalseAttributeInvalid = false ) { string configString = "Min_" + MinResults + "_Max_" + MaxResults + "_IoU_" + minTubeletIoUSimilarityThreshold + "_Ratio_" + ObjectCoverageApprovalThresholdPerVideo; Console.WriteLine("Aggregating for " + guid + " with param set " + configString); SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); List <SatyamResultsTableEntry> entries = resultsDB.getEntriesByGUID(guid); resultsDB.close(); SortedDictionary <DateTime, List <SatyamResultsTableEntry> > entriesBySubmitTime = SatyamResultValidationToolKit.SortResultsBySubmitTime_OneResultPerTurkerPerTask(entries); Dictionary <int, List <MultiObjectTrackingResult> > ResultsPerTask = new Dictionary <int, List <MultiObjectTrackingResult> >(); List <int> aggregatedTasks = new List <int>(); int noTotalConverged = 0; //int noCorrect = 0; int noTerminatedTasks = 0; List <SatyamAggregatedResultsTableEntry> aggEntries = new List <SatyamAggregatedResultsTableEntry>(); Dictionary <int, int> noResultsNeededForAggregation = SatyamResultsAnalysis.getNoResultsNeededForAggregationFromLog(configString, guid); Dictionary <int, int> noResultsNeededForAggregation_new = new Dictionary <int, int>(); // play back by time Dictionary <int, List <string> > WorkersPerTask = new Dictionary <int, List <string> >(); foreach (DateTime t in entriesBySubmitTime.Keys) { //Console.WriteLine("Processing Results of time: {0}", t); List <SatyamResultsTableEntry> ResultEntries = entriesBySubmitTime[t]; foreach (SatyamResultsTableEntry entry in ResultEntries) { SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString); MultiObjectTrackingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(task.jobEntry.JobParameters); string fileName = URIUtilities.filenameFromURINoExtension(task.SatyamURI); int taskEntryID = entry.SatyamTaskTableEntryID; if (aggregatedTasks.Contains(taskEntryID)) { continue; } if (!ResultsPerTask.ContainsKey(taskEntryID)) { ResultsPerTask.Add(taskEntryID, new List <MultiObjectTrackingResult>()); WorkersPerTask.Add(taskEntryID, new List <string>()); } // remove duplicate workers result string workerID = satyamResult.amazonInfo.WorkerID; if (WorkersPerTask[taskEntryID].Contains(workerID)) { continue; } //enclose only non-duplicate results, one per each worker. WorkersPerTask[taskEntryID].Add(workerID); string videoName = URIUtilities.localDirectoryNameFromURI(task.SatyamURI); string[] fields = videoName.Split('_'); int startingFrame = Convert.ToInt32(fields[fields.Length - 1]); int maxChunkEndFrame = startingFrame + job.ChunkDuration * job.FrameRate; int noFrameOverlap = (int)(job.ChunkOverlap * job.FrameRate); if (startingFrame != 0) { startingFrame -= noFrameOverlap; } string blobDir = URIUtilities.localDirectoryFullPathFromURI(task.SatyamURI); VATIC_DVA_CrowdsourcedResult taskr = TrackletLabelingAggregator.createVATIC_DVA_CrowdsourcedResultUsingSatyamBlobImageCount(satyamResult.TaskResult, blobDir, entry.SatyamTaskTableEntryID.ToString(), job.FrameRate); //MultiObjectTrackingResult result_tmp = vatic_tmp.getCompressedTracksInTimeSegment(); //VATIC_DVA_CrowdsourcedResult taskr = new VATIC_DVA_CrowdsourcedResult(satyamResult.TaskResult, videoName, entry.ID.ToString(), ImageURLs.Count, job.FrameRate); MultiObjectTrackingResult res = taskr.getCompressedTracksInTimeSegment(); if (allFalseAttributeInvalid) { if (TrackletLabelingAggregator.AllAttributeAllFalse(res)) { continue; } } ResultsPerTask[taskEntryID].Add(res); // check log if enough results are collected if (noResultsNeededForAggregation != null && noResultsNeededForAggregation.ContainsKey(taskEntryID) && ResultsPerTask[taskEntryID].Count < noResultsNeededForAggregation[taskEntryID]) { continue; } // hack for masters int tempMin = MinResults; if (TaskConstants.masterGUIDs.Contains(entry.JobGUID)) { tempMin = 1; } TrackletLabelingAggregatedResult aggResult = TrackletLabelingAggregator.getAggregatedResult(ResultsPerTask[taskEntryID], guid, task.SatyamURI, job.FrameRate, job.BoundaryLines, MinResults, MaxResults, boxToleranceThreshold, ObjectCoverageApprovalThresholdPerVideo, BoxCoverageApprovalThresholdPerTrack, consensusNumber, minTubeletIoUSimilarityThreshold, attributeMajority); if (aggResult == null) { continue; } //////////////// aggregation happen // record logs if (noResultsNeededForAggregation == null || !noResultsNeededForAggregation.ContainsKey(taskEntryID)) { noResultsNeededForAggregation_new.Add(taskEntryID, ResultsPerTask[taskEntryID].Count); } aggregatedTasks.Add(taskEntryID); noTotalConverged++; if (ResultsPerTask[taskEntryID].Count >= MaxResults) { noTerminatedTasks++; } SatyamAggregatedResult SatyamAggResult = new SatyamAggregatedResult(); SatyamAggResult.SatyamTaskTableEntryID = taskEntryID; SatyamAggResult.AggregatedResultString = JSonUtils.ConvertObjectToJSon <TrackletLabelingAggregatedResult>(aggResult); SatyamAggResult.TaskParameters = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString).TaskParametersString; SatyamAggregatedResultsTableEntry aggEntry = new SatyamAggregatedResultsTableEntry(); aggEntry.SatyamTaskTableEntryID = taskEntryID; aggEntry.JobGUID = entry.JobGUID; aggEntry.ResultString = JSonUtils.ConvertObjectToJSon <SatyamAggregatedResult>(SatyamAggResult); aggEntries.Add(aggEntry); } } Console.WriteLine("Total_Aggregated_Tasks: {0}", noTotalConverged); Console.WriteLine("Total_Terminated_Tasks: {0}", noTerminatedTasks); SatyamResultsAnalysis.RecordAggregationLog(noResultsNeededForAggregation_new, configString, guid); TrackletLabelingAnalyzer.GroupEntriesByVideoNameAndStitchAndSaveAggregatedResultVideosLocally(aggEntries, DirectoryConstants.defaultTempDirectory + guid); }
public static void AggregateWithParameterAndValidateSatyamImageNetClassificationResultByGUID(string guid, int MinResults = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MIN_RESULTS_TO_AGGREGATE, int MaxResults = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MAX_RESULTS_TO_AGGREGATE, double MajorityThreshold = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MAJORITY_THRESHOLD, string confusingImageListFilePath = null) { string configString = "Min_" + MinResults + "_Max_" + MaxResults + "_Thresh_" + MajorityThreshold; Console.WriteLine("Aggregating for param set " + configString); int noTerminatedTasks = 0; SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); List <SatyamResultsTableEntry> entries = resultsDB.getEntriesByGUIDOrderByID(guid); resultsDB.close(); //SortedDictionary<DateTime, List<SatyamResultsTableEntry>> entriesBySubmitTime = // SatyamResultValidation.SortResultsBySubmitTime_OneResultPerTurkerPerTask(entries); SortedDictionary <DateTime, List <SatyamResultsTableEntry> > entriesBySubmitTime = SatyamResultValidationToolKit.SortResultsBySubmitTime(entries); Dictionary <string, ConfusingReason> imageBlackListReason = new Dictionary <string, ConfusingReason>(); if (confusingImageListFilePath != null) { imageBlackListReason = getConfusingImageList(confusingImageListFilePath); } //Dictionary<int, List<SingleObjectLabelingResult>> ResultsPerTask = new Dictionary<int, List<SingleObjectLabelingResult>>(); Dictionary <int, List <SatyamResultsTableEntry> > ResultsPerTask = new Dictionary <int, List <SatyamResultsTableEntry> >(); List <int> aggregatedTasks = new List <int>(); int noTotalConverged = 0; int noCorrect = 0; SortedDictionary <string, Dictionary <string, int> > confusionMatrix_res_groundtruth = new SortedDictionary <string, Dictionary <string, int> >(); SortedDictionary <string, Dictionary <string, int> > confusionMatrix_groundtruth_res = new SortedDictionary <string, Dictionary <string, int> >(); StringBuilder s = new StringBuilder(); s.Append("<!DOCTYPE html>\n"); s.Append("<html>\n"); s.Append("<body>\n"); String title = String.Format("<h1>Job GUID {0} Incorrect Result Summary</h1>\n", guid); s.Append(title); Dictionary <int, int> noResultsNeededForAggregation = SatyamResultsAnalysis.getNoResultsNeededForAggregationFromLog(configString, guid); Dictionary <int, int> noResultsNeededForAggregation_new = new Dictionary <int, int>(); List <SatyamAggregatedResultsTableEntry> aggEntries = new List <SatyamAggregatedResultsTableEntry>(); foreach (DateTime t in entriesBySubmitTime.Keys) { foreach (SatyamResultsTableEntry entry in entriesBySubmitTime[t]) { //SatyamResultsTableEntry entry = entries[i]; SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString); SatyamJob job = task.jobEntry; string fileName = URIUtilities.filenameFromURI(task.SatyamURI); string imageCategoryName = fileName.Split('_')[0]; if (IsBlackListed(task.SatyamURI, imageBlackListReason)) { continue; } int taskEntryID = entry.SatyamTaskTableEntryID; if (aggregatedTasks.Contains(taskEntryID)) { continue; } if (!ResultsPerTask.ContainsKey(taskEntryID)) { ResultsPerTask.Add(taskEntryID, new List <SatyamResultsTableEntry>()); } //ResultEntriesPerTask[taskEntryID].Add(JSonUtils.ConvertJSonToObject<SingleObjectLabelingResult>(satyamResult.TaskResult)); ResultsPerTask[taskEntryID].Add(entry); // check log if enough results are collected if (noResultsNeededForAggregation != null && noResultsNeededForAggregation.ContainsKey(taskEntryID) && ResultsPerTask[taskEntryID].Count < noResultsNeededForAggregation[taskEntryID]) { continue; } string aggResultString = SingleObjectLabelingAggregator.GetAggregatedResultString(ResultsPerTask[taskEntryID], MinResults, MaxResults, MajorityThreshold); //SingleObjectLabelingAggregatedResult aggResult = SingleObjectLabelingAggregator.getAggregatedResult(ResultEntriesPerTask[taskEntryID], MinResults, MaxResults, MajorityThreshold); if (aggResultString == null) { continue; } SatyamAggregatedResultsTableEntry aggEntry = new SatyamAggregatedResultsTableEntry(); aggEntry.JobGUID = ResultsPerTask[taskEntryID][0].JobGUID; aggEntry.JobTemplateType = ResultsPerTask[taskEntryID][0].JobTemplateType; aggEntry.SatyamTaskTableEntryID = ResultsPerTask[taskEntryID][0].SatyamTaskTableEntryID; aggEntry.UserID = ResultsPerTask[taskEntryID][0].UserID; aggEntry.ResultString = aggResultString; /// aggregation happen // record logs if (noResultsNeededForAggregation == null || !noResultsNeededForAggregation.ContainsKey(taskEntryID)) { noResultsNeededForAggregation_new.Add(taskEntryID, ResultsPerTask[taskEntryID].Count); } /// //if (aggResult.Category == "None of the Above") //{ // continue; //} aggEntries.Add(aggEntry); SatyamSaveAggregatedDataSatyam data = new SatyamSaveAggregatedDataSatyam(aggEntry); String resultString = data.AggregatedResultString; SingleObjectLabelingAggregatedResult aggResult = JSonUtils.ConvertJSonToObject <SingleObjectLabelingAggregatedResult>(resultString); if (!confusionMatrix_res_groundtruth.ContainsKey(aggResult.Category)) { confusionMatrix_res_groundtruth.Add(aggResult.Category, new Dictionary <string, int>() { { GroundTruth[imageCategoryName], 0 } }); } else { if (!confusionMatrix_res_groundtruth[aggResult.Category].ContainsKey(GroundTruth[imageCategoryName])) { confusionMatrix_res_groundtruth[aggResult.Category].Add(GroundTruth[imageCategoryName], 0); } } if (!confusionMatrix_groundtruth_res.ContainsKey(GroundTruth[imageCategoryName])) { confusionMatrix_groundtruth_res.Add(GroundTruth[imageCategoryName], new Dictionary <string, int>() { { aggResult.Category, 0 } }); } else { if (!confusionMatrix_groundtruth_res[GroundTruth[imageCategoryName]].ContainsKey(aggResult.Category)) { confusionMatrix_groundtruth_res[GroundTruth[imageCategoryName]].Add(aggResult.Category, 0); } } if (aggResult.Category.Equals(GroundTruth[imageCategoryName], StringComparison.InvariantCultureIgnoreCase)) { noCorrect++; } else { //Console.WriteLine("{0}, Groundtruth: {1}, Aggregated: {2}, Votes: {3}", // fileName, GroundTruth[imageCategoryName], aggResult.Category, // JSonUtils.ConvertObjectToJSon(aggResult.metaData)); String record = String.Format("<p>{0}, Groundtruth: {1}, Aggregated: {2}, Votes: {3}</p>\n", fileName, GroundTruth[imageCategoryName], aggResult.Category, JSonUtils.ConvertObjectToJSon(aggResult.metaData)); String img = String.Format("<img src=\"{0}\" >\n", task.SatyamURI); s.Append(record); s.Append(img); } noTotalConverged++; if (ResultsPerTask[taskEntryID].Count >= MaxResults) { noTerminatedTasks++; } confusionMatrix_res_groundtruth[aggResult.Category][GroundTruth[imageCategoryName]]++; confusionMatrix_groundtruth_res[GroundTruth[imageCategoryName]][aggResult.Category]++; aggregatedTasks.Add(taskEntryID); } } SatyamResultsAnalysis.RecordAggregationLog(noResultsNeededForAggregation_new, configString, guid); s.Append("</body>\n"); s.Append("</html>\n"); string dataToBeSaved = s.ToString(); SatyamJobStorageAccountAccess storage = new SatyamJobStorageAccountAccess(); string FileName = String.Format("AggregatedIncorrectResults-{0}_Min{1}Max{2}Thresh{3}.html", guid, MinResults, MaxResults, MajorityThreshold); storage.SaveATextFile("singleobjectlabeling", guid, FileName, dataToBeSaved); s.Clear(); s.Append("<!DOCTYPE html>\n"); s.Append("<html>\n"); s.Append("<body>\n"); s.Append(title); string resultSummary = String.Format("<p>Result: {0}/{1}, precision: {2}</p>\n", noCorrect, noTotalConverged, (double)noCorrect / noTotalConverged); resultSummary += String.Format("<p>Terminated: {0}, Not Enough Results: {1}</p>\n", noTerminatedTasks, ResultsPerTask.Count - noTotalConverged); Console.WriteLine(resultSummary); s.Append(resultSummary); // write the confusion matrix s.Append("<p>"); String row = "\t\t"; foreach (string resultCategory in confusionMatrix_res_groundtruth.Keys) { row += resultCategory + "\t"; } row += "<br>\n"; s.Append(row); Console.WriteLine(row); string matString = ""; foreach (string groundTruthCategory in confusionMatrix_groundtruth_res.Keys) { row = groundTruthCategory + "\t"; foreach (string resultCategory in confusionMatrix_res_groundtruth.Keys) { if (confusionMatrix_groundtruth_res[groundTruthCategory].ContainsKey(resultCategory)) { row += confusionMatrix_groundtruth_res[groundTruthCategory][resultCategory].ToString(); } else { row += "0"; } row += "\t"; } row += "<br>\n"; s.Append(row); Console.WriteLine(row); matString += row; } s.Append("</p>\n"); s.Append("</body>\n"); s.Append("</html>\n"); string summaryToBeSaved = s.ToString(); FileName = String.Format("Aggregated_Summary-{0}_Min{1}Max{2}Thresh{3}.html", guid, MinResults, MaxResults, MajorityThreshold); storage.SaveATextFile("singleobjectlabeling", guid, FileName, summaryToBeSaved); /// local file string outputString = String.Format("{0} {1} {2} {3} {4} {5}\n", configString, noCorrect, noTotalConverged, (double)noCorrect / noTotalConverged, noTerminatedTasks, ResultsPerTask.Count - noTotalConverged); string outputfile = DirectoryConstants.defaultTempDirectory + guid + "\\resultSummary.txt"; File.AppendAllText(outputfile, outputString); string outputmatFile = DirectoryConstants.defaultTempDirectory + guid + "\\" + configString + "_mat.txt"; File.WriteAllText(outputmatFile, matString); //for (double prob = 0; prob < 1;prob +=0.2) //{ // SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entries, guid, configString, anotherChanceProbablity: prob); //} //for (double ratio = 0; ratio < 1; ratio += 0.2) //{ // SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guid, configString, approvalRatioThreshold: ratio); //} SatyamResultsAnalysis.AggregationAnalysis(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guid, configString); }
public static void AggregateWithParameterAndValidateObjectCountingResultByGUID(string guid, //bool saveImage = false, bool prepareTrainingSet = false, string outputDirectory = null, int MinHeight = TaskConstants.OBJECT_COUNTING_VALIDATION_MIN_HEIGHT, int MaxOcclusion = TaskConstants.OBJECT_COUNTING_VALIDATION_MAX_OCCLUSION, double Max_Truncation = TaskConstants.OBJECT_COUNTING_VALIDATION_MIN_TRUNCATION, int MinResults = TaskConstants.OBJECT_COUNTING_MTURK_MIN_RESULTS_TO_AGGREGATE, int MaxResults = TaskConstants.OBJECT_COUNTING_MTURK_MAX_RESULTS_TO_AGGREGATE, double MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND = TaskConstants.OBJECT_COUNTING_MTURK_MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND, double MAX_DEVIATION_FRACTION = TaskConstants.OBJECT_COUNTING_MTURK_MAX_DEVIATION_FRACTION, double SUPER_MAJORITY_VALUE = TaskConstants.OBJECT_COUNTING_MTURK_SUPER_MAJORITY_VALUE ) { if (!Directory.Exists(DirectoryConstants.defaultTempDirectory + guid)) { Directory.CreateDirectory(DirectoryConstants.defaultTempDirectory + guid); } string configString = "Min_" + MinResults + "_Max_" + MaxResults + "_Dev_" + MAX_DEVIATION_FRACTION + "_Majority_" + SUPER_MAJORITY_VALUE; Console.WriteLine("Aggregating for param set " + configString); int noTerminatedTasks = 0; SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); List <SatyamResultsTableEntry> entries = resultsDB.getEntriesByGUIDOrderByID(guid); resultsDB.close(); SortedDictionary <DateTime, List <SatyamResultsTableEntry> > entriesBySubmitTime = SatyamResultValidationToolKit.SortResultsBySubmitTime(entries); int noTotalConverged = 0; //int noCorrect = 0; Dictionary <int, List <SatyamResultsTableEntry> > ResultsPerTask = new Dictionary <int, List <SatyamResultsTableEntry> >(); List <int> aggregatedTasks = new List <int>(); List <SatyamAggregatedResultsTableEntry> aggEntries = new List <SatyamAggregatedResultsTableEntry>(); Dictionary <int, int> noResultsNeededForAggregation = SatyamResultsAnalysis.getNoResultsNeededForAggregationFromLog(configString, guid); Dictionary <int, int> noResultsNeededForAggregation_new = new Dictionary <int, int>(); // play back by time foreach (DateTime t in entriesBySubmitTime.Keys) { //Console.WriteLine("Processing Results of time: {0}", t); foreach (SatyamResultsTableEntry entry in entriesBySubmitTime[t]) { SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString); SatyamJob job = task.jobEntry; string fileName = URIUtilities.filenameFromURI(task.SatyamURI); int taskEntryID = entry.SatyamTaskTableEntryID; if (aggregatedTasks.Contains(taskEntryID)) { continue; } if (!ResultsPerTask.ContainsKey(taskEntryID)) { ResultsPerTask.Add(taskEntryID, new List <SatyamResultsTableEntry>()); } ResultsPerTask[taskEntryID].Add(entry); // check log if enough results are collected if (noResultsNeededForAggregation != null && noResultsNeededForAggregation.ContainsKey(taskEntryID) && ResultsPerTask[taskEntryID].Count < noResultsNeededForAggregation[taskEntryID]) { continue; } string aggResultString = ObjectCountingAggregator.GetAggregatedResultString(ResultsPerTask[taskEntryID], MinResults, MaxResults, MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND, MAX_DEVIATION_FRACTION, SUPER_MAJORITY_VALUE); if (aggResultString == null) { continue; } SatyamAggregatedResultsTableEntry aggEntry = new SatyamAggregatedResultsTableEntry(); aggEntry.JobGUID = ResultsPerTask[taskEntryID][0].JobGUID; aggEntry.JobTemplateType = ResultsPerTask[taskEntryID][0].JobTemplateType; aggEntry.SatyamTaskTableEntryID = taskEntryID; aggEntry.UserID = ResultsPerTask[taskEntryID][0].UserID; aggEntry.ResultString = aggResultString; /// aggregation happen /// // record logs // record logs if (noResultsNeededForAggregation == null || !noResultsNeededForAggregation.ContainsKey(taskEntryID)) { noResultsNeededForAggregation_new.Add(taskEntryID, ResultsPerTask[taskEntryID].Count); } aggEntries.Add(aggEntry); noTotalConverged++; if (ResultsPerTask[taskEntryID].Count >= MaxResults) { noTerminatedTasks++; } aggregatedTasks.Add(taskEntryID); } } SatyamResultsAnalysis.RecordAggregationLog(noResultsNeededForAggregation_new, configString, guid); double totalError, totalGroundtruth; //ValidateSatyamKITTIObjectCountingAggregationResult(aggEntries, out totalError, out totalGroundtruth, // MinHeight, MaxOcclusion, Max_Truncation); ValidateSatyamCARPKObjectCountingAggregationResult(aggEntries, out totalError, out totalGroundtruth); Console.WriteLine("noTotalConverged {0}", noTotalConverged); Console.WriteLine("noTerminatedTasks {0}", noTerminatedTasks); /// local file string outputString = String.Format("{0} {1} {2} {3} {4} {5} {6}\n", configString, totalError, totalGroundtruth, totalError / totalGroundtruth, noTotalConverged, noTerminatedTasks, ResultsPerTask.Count - noTotalConverged); string outputfile = DirectoryConstants.defaultTempDirectory + guid + "\\resultSummary.txt"; File.AppendAllText(outputfile, outputString); string approvalString = configString + "_PayDev_" + TaskConstants.OBJECT_COUNTING_MTURK_MAX_ABSOLUTE_COUNT_DEVIATION_FOR_PAYMENT + "_PayFrac_" + TaskConstants.OBJECT_COUNTING_MTURK_MAX_DEVIATION_FRACTION_FOR_PAYMENT; //for (double prob = 0; prob < 1;prob +=0.2) //{ // SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entries, guid, approvalString, anotherChanceProbablity: prob); //} //for (double ratio = 0; ratio < 1; ratio += 0.2) //{ // SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guid, configString, approvalRatioThreshold: ratio); //} SatyamResultsAnalysis.AggregationAnalysis(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guid, configString); }
/// <summary> /// function generalizable.. TODO... to all tasks /// </summary> /// <param name="guids"></param> /// <param name="IoUTreshold"></param> /// <param name="saveImage"></param> /// <param name="outputDirectory"></param> /// <param name="MinResults"></param> /// <param name="MaxResults"></param> /// <param name="CategoryMajorityThreshold"></param> /// <param name="PolygonBoundaryMajorityThreshold"></param> /// <param name="ObjectsCoverageThreshold"></param> /// <param name="overwrite"></param> /// <param name="approvalAnalysis"></param> public static void AggregateWithParameterAndValidatePascalVOCImageSegmentationByGUID(List <string> guids, double IoUTreshold, bool saveImage = false, string outputDirectory = null, int MinResults = TaskConstants.IMAGE_SEGMENTATION_MTURK_MIN_RESULTS_TO_AGGREGATE, int MaxResults = TaskConstants.IMAGE_SEGMENTATION_MTURK_MAX_RESULTS_TO_AGGREGATE, double CategoryMajorityThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_MAJORITY_CATEGORY_THRESHOLD, double PolygonBoundaryMajorityThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_MAJORITY_POLYGON_BOUNDARY_THRESHOLD, double ObjectsCoverageThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_OBJECT_COVERAGE_THRESHOLD_FOR_AGGREGATION_TERMINATION, double minSimilarityThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_POLYGON_IOU_THRESHOLD, int minResultsForConsensus = TaskConstants.IMAGE_SEGMENTATION_MTURK_MIN_RESULTS_FOR_CONSENSUS, bool overwrite = false, bool approvalAnalysis = false) { string configString = "Min_" + MinResults + "_Max_" + MaxResults + "_Majority_" + PolygonBoundaryMajorityThreshold + "_Ratio_" + ObjectsCoverageThreshold; Console.WriteLine("Aggregating for param set " + configString); if (!overwrite && File.Exists(DirectoryConstants.defaultTempDirectory + "\\ImageSegmentationResult\\" + configString + ".txt")) { return; } SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); Dictionary <int, List <string> > WorkersPerTask = new Dictionary <int, List <string> >(); List <SatyamResultsTableEntry> entries = new List <SatyamResultsTableEntry>(); foreach (string guid in guids) { entries.AddRange(resultsDB.getEntriesByGUIDOrderByID(guid)); } resultsDB.close(); SortedDictionary <DateTime, List <SatyamResultsTableEntry> > entriesBySubmitTime = SatyamResultValidationToolKit.SortResultsBySubmitTime_OneResultPerTurkerPerTask(entries); Dictionary <int, List <ImageSegmentationResult> > ResultsPerTask = new Dictionary <int, List <ImageSegmentationResult> >(); List <int> aggregatedTasks = new List <int>(); int noTotalConverged = 0; //int noCorrect = 0; int noTerminatedTasks = 0; List <SatyamAggregatedResultsTableEntry> aggEntries = new List <SatyamAggregatedResultsTableEntry>(); Dictionary <int, int> noResultsNeededForAggregation = SatyamResultsAnalysis.getNoResultsNeededForAggregationFromLog(configString, guids[0]); Dictionary <int, int> noResultsNeededForAggregation_new = new Dictionary <int, int>(); // play back by time foreach (DateTime t in entriesBySubmitTime.Keys) { //Console.WriteLine("Processing Results of time: {0}", t); List <SatyamResultsTableEntry> ResultEntries = entriesBySubmitTime[t]; foreach (SatyamResultsTableEntry entry in ResultEntries) { SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString); SatyamJob job = task.jobEntry; string fileName = URIUtilities.filenameFromURINoExtension(task.SatyamURI); int taskEntryID = entry.SatyamTaskTableEntryID; if (aggregatedTasks.Contains(taskEntryID)) { continue; } if (!ResultsPerTask.ContainsKey(taskEntryID)) { ResultsPerTask.Add(taskEntryID, new List <ImageSegmentationResult>()); } ResultsPerTask[taskEntryID].Add(JSonUtils.ConvertJSonToObject <ImageSegmentationResult>(satyamResult.TaskResult)); // check log if enough results are collected if (noResultsNeededForAggregation != null && noResultsNeededForAggregation.ContainsKey(taskEntryID) && ResultsPerTask[taskEntryID].Count < noResultsNeededForAggregation[taskEntryID]) { continue; } ImageSegmentationAggregatedResult aggResult = ImageSegmentationAggregator.getAggregatedResult( ResultsPerTask[taskEntryID], task.SatyamURI, entry.JobGUID, MinResults, MaxResults, CategoryMajorityThreshold, PolygonBoundaryMajorityThreshold, ObjectsCoverageThreshold, minSimilarityThreshold, minResultsForConsensus); if (aggResult == null) { continue; } // aggregation happen // record logs if (noResultsNeededForAggregation == null || !noResultsNeededForAggregation.ContainsKey(taskEntryID)) { noResultsNeededForAggregation_new.Add(taskEntryID, ResultsPerTask[taskEntryID].Count); } aggregatedTasks.Add(taskEntryID); noTotalConverged++; if (ResultsPerTask[taskEntryID].Count >= MaxResults) { noTerminatedTasks++; } SatyamAggregatedResult SatyamAggResult = new SatyamAggregatedResult(); SatyamAggResult.SatyamTaskTableEntryID = taskEntryID; SatyamAggResult.AggregatedResultString = JSonUtils.ConvertObjectToJSon <ImageSegmentationAggregatedResult>(aggResult); SatyamAggResult.TaskParameters = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString).TaskParametersString; SatyamAggregatedResultsTableEntry aggEntry = new SatyamAggregatedResultsTableEntry(); aggEntry.SatyamTaskTableEntryID = taskEntryID; aggEntry.JobGUID = entry.JobGUID; aggEntry.ResultString = JSonUtils.ConvertObjectToJSon <SatyamAggregatedResult>(SatyamAggResult); aggEntries.Add(aggEntry); List <SatyamAggregatedResultsTableEntry> tmpEntries = new List <SatyamAggregatedResultsTableEntry>(); tmpEntries.Add(aggEntry); //ValidateSatyamKITTIDetectionAggregationResult(tmpEntries, saveImage, MinHeight, MaxOcclusion, Max_Truncation); } } Console.WriteLine("Total_Aggregated_Tasks: {0}", noTotalConverged); Console.WriteLine("Total_Terminated_Tasks: {0}", noTerminatedTasks); SatyamResultsAnalysis.RecordAggregationLog(noResultsNeededForAggregation_new, configString, guids[0]); string r = ValidatePascalVOCImageSegmentationResult_InstanceLevel(aggEntries, IoUTreshold); r = noTotalConverged + " " + noTerminatedTasks + " " + r; File.WriteAllText(DirectoryConstants.defaultTempDirectory + "\\ImageSegmentationResult\\" + configString + ".txt", r); if (approvalAnalysis) { string approvalString = configString + "_PayCover_" + TaskConstants.IMAGE_SEGMENTATION_MTURK_OBJECT_COVERAGE_THRESHOLD_FOR_PAYMENT + "_PayIoU_" + TaskConstants.IMAGE_SEGMENTATION_MTURK_POLYGON_IOU_THRESHOLD_FOR_PAYMENT; //for (double ratio = 0; ratio < 1; ratio += 0.2) //{ // SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guids[0], configString, approvalRatioThreshold: ratio); //} SatyamResultsAnalysis.AggregationAnalysis(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guids[0], configString); } }