コード例 #1
0
        /// <summary>
        /// function generalizable.. TODO... to all tasks
        /// </summary>
        /// <param name="guids"></param>
        /// <param name="IoUTreshold"></param>
        /// <param name="saveImage"></param>
        /// <param name="outputDirectory"></param>
        /// <param name="MinResults"></param>
        /// <param name="MaxResults"></param>
        /// <param name="CategoryMajorityThreshold"></param>
        /// <param name="PolygonBoundaryMajorityThreshold"></param>
        /// <param name="ObjectsCoverageThreshold"></param>
        /// <param name="overwrite"></param>
        /// <param name="approvalAnalysis"></param>
        public static void AggregateWithParameterAndValidatePascalVOCImageSegmentationByGUID(List <string> guids,
                                                                                             double IoUTreshold,
                                                                                             bool saveImage = false, string outputDirectory = null,
                                                                                             int MinResults = TaskConstants.IMAGE_SEGMENTATION_MTURK_MIN_RESULTS_TO_AGGREGATE,
                                                                                             int MaxResults = TaskConstants.IMAGE_SEGMENTATION_MTURK_MAX_RESULTS_TO_AGGREGATE,
                                                                                             double CategoryMajorityThreshold        = TaskConstants.IMAGE_SEGMENTATION_MTURK_MAJORITY_CATEGORY_THRESHOLD,
                                                                                             double PolygonBoundaryMajorityThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_MAJORITY_POLYGON_BOUNDARY_THRESHOLD,
                                                                                             double ObjectsCoverageThreshold         = TaskConstants.IMAGE_SEGMENTATION_MTURK_OBJECT_COVERAGE_THRESHOLD_FOR_AGGREGATION_TERMINATION,
                                                                                             double minSimilarityThreshold           = TaskConstants.IMAGE_SEGMENTATION_MTURK_POLYGON_IOU_THRESHOLD,
                                                                                             int minResultsForConsensus = TaskConstants.IMAGE_SEGMENTATION_MTURK_MIN_RESULTS_FOR_CONSENSUS,
                                                                                             bool overwrite             = false,
                                                                                             bool approvalAnalysis      = false)
        {
            string configString = "Min_" + MinResults + "_Max_" + MaxResults + "_Majority_" + PolygonBoundaryMajorityThreshold + "_Ratio_" + ObjectsCoverageThreshold;

            Console.WriteLine("Aggregating for param set " + configString);
            if (!overwrite && File.Exists(DirectoryConstants.defaultTempDirectory + "\\ImageSegmentationResult\\" + configString + ".txt"))
            {
                return;
            }

            SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();

            Dictionary <int, List <string> > WorkersPerTask = new Dictionary <int, List <string> >();

            List <SatyamResultsTableEntry> entries = new List <SatyamResultsTableEntry>();

            foreach (string guid in guids)
            {
                entries.AddRange(resultsDB.getEntriesByGUIDOrderByID(guid));
            }
            resultsDB.close();

            SortedDictionary <DateTime, List <SatyamResultsTableEntry> > entriesBySubmitTime =
                SatyamResultValidationToolKit.SortResultsBySubmitTime_OneResultPerTurkerPerTask(entries);

            Dictionary <int, List <ImageSegmentationResult> > ResultsPerTask = new Dictionary <int, List <ImageSegmentationResult> >();
            List <int> aggregatedTasks = new List <int>();

            int noTotalConverged = 0;
            //int noCorrect = 0;
            int noTerminatedTasks = 0;

            List <SatyamAggregatedResultsTableEntry> aggEntries     = new List <SatyamAggregatedResultsTableEntry>();
            Dictionary <int, int> noResultsNeededForAggregation     = SatyamResultsAnalysis.getNoResultsNeededForAggregationFromLog(configString, guids[0]);
            Dictionary <int, int> noResultsNeededForAggregation_new = new Dictionary <int, int>();

            // play back by time
            foreach (DateTime t in entriesBySubmitTime.Keys)
            {
                //Console.WriteLine("Processing Results of time: {0}", t);
                List <SatyamResultsTableEntry> ResultEntries = entriesBySubmitTime[t];
                foreach (SatyamResultsTableEntry entry in ResultEntries)
                {
                    SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
                    SatyamTask   task         = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
                    SatyamJob    job          = task.jobEntry;
                    string       fileName     = URIUtilities.filenameFromURINoExtension(task.SatyamURI);
                    int          taskEntryID  = entry.SatyamTaskTableEntryID;
                    if (aggregatedTasks.Contains(taskEntryID))
                    {
                        continue;
                    }
                    if (!ResultsPerTask.ContainsKey(taskEntryID))
                    {
                        ResultsPerTask.Add(taskEntryID, new List <ImageSegmentationResult>());
                    }

                    ResultsPerTask[taskEntryID].Add(JSonUtils.ConvertJSonToObject <ImageSegmentationResult>(satyamResult.TaskResult));

                    // check log if enough results are collected
                    if (noResultsNeededForAggregation != null && noResultsNeededForAggregation.ContainsKey(taskEntryID) &&
                        ResultsPerTask[taskEntryID].Count < noResultsNeededForAggregation[taskEntryID])
                    {
                        continue;
                    }

                    ImageSegmentationAggregatedResult aggResult = ImageSegmentationAggregator.getAggregatedResult(
                        ResultsPerTask[taskEntryID], task.SatyamURI, entry.JobGUID,
                        MinResults, MaxResults, CategoryMajorityThreshold,
                        PolygonBoundaryMajorityThreshold, ObjectsCoverageThreshold,
                        minSimilarityThreshold, minResultsForConsensus);
                    if (aggResult == null)
                    {
                        continue;
                    }

                    // aggregation happen
                    // record logs
                    if (noResultsNeededForAggregation == null || !noResultsNeededForAggregation.ContainsKey(taskEntryID))
                    {
                        noResultsNeededForAggregation_new.Add(taskEntryID, ResultsPerTask[taskEntryID].Count);
                    }
                    aggregatedTasks.Add(taskEntryID);
                    noTotalConverged++;
                    if (ResultsPerTask[taskEntryID].Count >= MaxResults)
                    {
                        noTerminatedTasks++;
                    }
                    SatyamAggregatedResult SatyamAggResult = new SatyamAggregatedResult();
                    SatyamAggResult.SatyamTaskTableEntryID = taskEntryID;
                    SatyamAggResult.AggregatedResultString = JSonUtils.ConvertObjectToJSon <ImageSegmentationAggregatedResult>(aggResult);
                    SatyamAggResult.TaskParameters         = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString).TaskParametersString;

                    SatyamAggregatedResultsTableEntry aggEntry = new SatyamAggregatedResultsTableEntry();
                    aggEntry.SatyamTaskTableEntryID = taskEntryID;
                    aggEntry.JobGUID      = entry.JobGUID;
                    aggEntry.ResultString = JSonUtils.ConvertObjectToJSon <SatyamAggregatedResult>(SatyamAggResult);


                    aggEntries.Add(aggEntry);
                    List <SatyamAggregatedResultsTableEntry> tmpEntries = new List <SatyamAggregatedResultsTableEntry>();
                    tmpEntries.Add(aggEntry);

                    //ValidateSatyamKITTIDetectionAggregationResult(tmpEntries, saveImage, MinHeight, MaxOcclusion, Max_Truncation);
                }
            }
            Console.WriteLine("Total_Aggregated_Tasks: {0}", noTotalConverged);
            Console.WriteLine("Total_Terminated_Tasks: {0}", noTerminatedTasks);

            SatyamResultsAnalysis.RecordAggregationLog(noResultsNeededForAggregation_new, configString, guids[0]);

            string r = ValidatePascalVOCImageSegmentationResult_InstanceLevel(aggEntries, IoUTreshold);

            r = noTotalConverged + " " + noTerminatedTasks + " " + r;
            File.WriteAllText(DirectoryConstants.defaultTempDirectory + "\\ImageSegmentationResult\\" + configString + ".txt", r);


            if (approvalAnalysis)
            {
                string approvalString = configString + "_PayCover_" + TaskConstants.IMAGE_SEGMENTATION_MTURK_OBJECT_COVERAGE_THRESHOLD_FOR_PAYMENT +
                                        "_PayIoU_" + TaskConstants.IMAGE_SEGMENTATION_MTURK_POLYGON_IOU_THRESHOLD_FOR_PAYMENT;
                //for (double ratio = 0; ratio < 1; ratio += 0.2)
                //{
                //    SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guids[0], configString, approvalRatioThreshold: ratio);
                //}
                SatyamResultsAnalysis.AggregationAnalysis(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guids[0], configString);
            }
        }
コード例 #2
0
        public static void StaticOfflineAggregationWithParameterAndValidation(List <string> guids,
                                                                              double IoUTreshold,
                                                                              int MinResults = TaskConstants.IMAGE_SEGMENTATION_MTURK_MIN_RESULTS_TO_AGGREGATE,
                                                                              int MaxResults = TaskConstants.IMAGE_SEGMENTATION_MTURK_MAX_RESULTS_TO_AGGREGATE,
                                                                              double CategoryMajorityThreshold        = TaskConstants.IMAGE_SEGMENTATION_MTURK_MAJORITY_CATEGORY_THRESHOLD,
                                                                              double PolygonBoundaryMajorityThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_MAJORITY_POLYGON_BOUNDARY_THRESHOLD,
                                                                              double ObjectsCoverageThreshold         = TaskConstants.IMAGE_SEGMENTATION_MTURK_OBJECT_COVERAGE_THRESHOLD_FOR_AGGREGATION_TERMINATION,
                                                                              double minSimilarityThreshold           = TaskConstants.IMAGE_SEGMENTATION_MTURK_POLYGON_IOU_THRESHOLD,
                                                                              int minResultsForConsensus = TaskConstants.IMAGE_SEGMENTATION_MTURK_MIN_RESULTS_FOR_CONSENSUS)
        {
            SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();

            Dictionary <int, List <string> > WorkersPerTask = new Dictionary <int, List <string> >();

            List <SatyamResultsTableEntry> entries = new List <SatyamResultsTableEntry>();

            foreach (string guid in guids)
            {
                entries.AddRange(resultsDB.getEntriesByGUIDOrderByID(guid));
            }
            resultsDB.close();

            SortedDictionary <int, List <SatyamResultsTableEntry> > EntriesPerTask = new SortedDictionary <int, List <SatyamResultsTableEntry> >();
            SortedDictionary <int, List <ImageSegmentationResult> > ResultsPerTask = new SortedDictionary <int, List <ImageSegmentationResult> >();
            List <int> aggregatedTasks = new List <int>();

            int noTotalConverged = 0;
            //int noCorrect = 0;
            int noTerminatedTasks = 0;

            List <SatyamAggregatedResultsTableEntry> aggEntries = new List <SatyamAggregatedResultsTableEntry>();

            // Organize by taskID
            foreach (SatyamResultsTableEntry entry in entries)
            {
                SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
                int          taskEntryID  = entry.SatyamTaskTableEntryID;

                if (!EntriesPerTask.ContainsKey(taskEntryID))
                {
                    EntriesPerTask.Add(taskEntryID, new List <SatyamResultsTableEntry>());
                    ResultsPerTask.Add(taskEntryID, new List <ImageSegmentationResult>());
                }
                EntriesPerTask[taskEntryID].Add(entry);
                ResultsPerTask[taskEntryID].Add(JSonUtils.ConvertJSonToObject <ImageSegmentationResult>(satyamResult.TaskResult));
            }
            foreach (int taskEntryID in EntriesPerTask.Keys)
            {
                SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(EntriesPerTask[taskEntryID][0].ResultString);
                SatyamTask   task         = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
                SatyamJob    job          = task.jobEntry;
                string       fileName     = URIUtilities.filenameFromURINoExtension(task.SatyamURI);

                List <int> taskidfilter = new List <int>()
                {
                    40430,
                    40432,
                    40433,
                    40434,
                    40440,
                    40447,
                    40451,
                    40460,
                };

                //if (fileName != "2007_000549") continue;

                if (!taskidfilter.Contains(satyamResult.TaskTableEntryID))
                {
                    continue;
                }

                Console.WriteLine("Aggregating task {0}: {1} results", taskEntryID, EntriesPerTask[taskEntryID].Count);
                ImageSegmentationAggregatedResult aggResult = ImageSegmentationAggregator.getAggregatedResult(
                    ResultsPerTask[taskEntryID], task.SatyamURI, job.JobGUIDString,
                    MinResults, MaxResults, CategoryMajorityThreshold,
                    PolygonBoundaryMajorityThreshold, ObjectsCoverageThreshold,
                    minSimilarityThreshold, minResultsForConsensus);
                if (aggResult == null)
                {
                    continue;
                }

                // aggregation happen

                aggregatedTasks.Add(taskEntryID);
                noTotalConverged++;
                if (ResultsPerTask[taskEntryID].Count >= MaxResults)
                {
                    noTerminatedTasks++;
                }
                SatyamAggregatedResult SatyamAggResult = new SatyamAggregatedResult();
                SatyamAggResult.SatyamTaskTableEntryID = taskEntryID;
                SatyamAggResult.AggregatedResultString = JSonUtils.ConvertObjectToJSon <ImageSegmentationAggregatedResult>(aggResult);
                SatyamAggResult.TaskParameters         = satyamResult.TaskParametersString;

                SatyamAggregatedResultsTableEntry aggEntry = new SatyamAggregatedResultsTableEntry();
                aggEntry.SatyamTaskTableEntryID = taskEntryID;
                aggEntry.JobGUID      = job.JobGUIDString;
                aggEntry.ResultString = JSonUtils.ConvertObjectToJSon <SatyamAggregatedResult>(SatyamAggResult);


                aggEntries.Add(aggEntry);
                List <SatyamAggregatedResultsTableEntry> tmpEntries = new List <SatyamAggregatedResultsTableEntry>();
                tmpEntries.Add(aggEntry);

                ValidatePascalVOCImageSegmentationResult_InstanceLevel(tmpEntries, IoUTreshold);
            }
            Console.WriteLine("Total_Aggregated_Tasks: {0}", noTotalConverged);
            Console.WriteLine("Total_Terminated_Tasks: {0}", noTerminatedTasks);

            string r = ValidatePascalVOCImageSegmentationResult_InstanceLevel(aggEntries, IoUTreshold);
        }