public static void AggregateWithParameterAndValidateSatyamVideoClassificationResultByGUID(string guid,
                                                                                                  int MinResults                    = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MIN_RESULTS_TO_AGGREGATE,
                                                                                                  int MaxResults                    = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MAX_RESULTS_TO_AGGREGATE,
                                                                                                  double MajorityThreshold          = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MAJORITY_THRESHOLD,
                                                                                                  bool prepareDataForTraining       = false, string outputDirectory = null,
                                                                                                  string confusingVideoListFilePath = null)
        {
            string configString = "Min_" + MinResults + "_Max_" + MaxResults + "_Thresh_" + MajorityThreshold;

            Console.WriteLine("Aggregating for param set " + configString);
            int noTerminatedTasks = 0;
            SatyamResultsTableAccess       resultsDB = new SatyamResultsTableAccess();
            List <SatyamResultsTableEntry> entries   = resultsDB.getEntriesByGUIDOrderByID(guid);

            resultsDB.close();


            SortedDictionary <DateTime, List <SatyamResultsTableEntry> > entriesBySubmitTime = SatyamResultValidationToolKit.SortResultsBySubmitTime(entries);

            int noTotalConverged = 0;
            int noCorrect        = 0;

            Dictionary <int, List <SatyamResultsTableEntry> > ResultsPerTask = new Dictionary <int, List <SatyamResultsTableEntry> >();
            List <int> aggregatedTasks = new List <int>();

            SortedDictionary <string, Dictionary <string, int> > confusionMatrix_res_groundtruth = new SortedDictionary <string, Dictionary <string, int> >();
            SortedDictionary <string, Dictionary <string, int> > confusionMatrix_groundtruth_res = new SortedDictionary <string, Dictionary <string, int> >();

            List <SatyamAggregatedResultsTableEntry> aggEntries = new List <SatyamAggregatedResultsTableEntry>();

            Dictionary <int, int> noResultsNeededForAggregation     = SatyamResultsAnalysis.getNoResultsNeededForAggregationFromLog(configString, guid);
            Dictionary <int, int> noResultsNeededForAggregation_new = new Dictionary <int, int>();

            foreach (DateTime t in entriesBySubmitTime.Keys)
            {
                //Console.WriteLine("Processing Results of time: {0}", t);
                foreach (SatyamResultsTableEntry entry in entriesBySubmitTime[t])
                {
                    SatyamResult satyamResult             = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
                    SatyamTask   task                     = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
                    SatyamJob    job                      = task.jobEntry;
                    string       fileName                 = URIUtilities.filenameFromURI(task.SatyamURI);
                    string       VideoCategoryGroundTruth = getVideoCategoryFromFileName(fileName);

                    int taskEntryID = entry.SatyamTaskTableEntryID;
                    if (aggregatedTasks.Contains(taskEntryID))
                    {
                        continue;
                    }

                    if (!ResultsPerTask.ContainsKey(taskEntryID))
                    {
                        ResultsPerTask.Add(taskEntryID, new List <SatyamResultsTableEntry>());
                    }

                    ResultsPerTask[taskEntryID].Add(entry);

                    // check log if enough results are collected
                    if (noResultsNeededForAggregation != null && noResultsNeededForAggregation.ContainsKey(taskEntryID) &&
                        ResultsPerTask[taskEntryID].Count < noResultsNeededForAggregation[taskEntryID])
                    {
                        continue;
                    }

                    //SingleObjectLabelingAggregatedResult aggResult = SingleObjectLabelingAggregator.getAggregatedResult(ResultsPerTask[taskEntryID], MinResults, MaxResult, MajorityThreshold);
                    string aggResultString = SingleObjectLabelingAggregator.GetAggregatedResultString(ResultsPerTask[taskEntryID], MinResults, MaxResults, MajorityThreshold);

                    if (aggResultString == null)
                    {
                        continue;
                    }

                    SatyamAggregatedResultsTableEntry aggEntry = new SatyamAggregatedResultsTableEntry();
                    aggEntry.JobGUID                = ResultsPerTask[taskEntryID][0].JobGUID;
                    aggEntry.JobTemplateType        = ResultsPerTask[taskEntryID][0].JobTemplateType;
                    aggEntry.SatyamTaskTableEntryID = taskEntryID;
                    aggEntry.UserID       = ResultsPerTask[taskEntryID][0].UserID;
                    aggEntry.ResultString = aggResultString;

                    /// aggregation happen
                    // record logs
                    if (noResultsNeededForAggregation == null || !noResultsNeededForAggregation.ContainsKey(taskEntryID))
                    {
                        noResultsNeededForAggregation_new.Add(taskEntryID, ResultsPerTask[taskEntryID].Count);
                    }

                    aggEntries.Add(aggEntry);
                    noTotalConverged++;
                    if (ResultsPerTask[taskEntryID].Count >= MaxResults)
                    {
                        noTerminatedTasks++;
                    }
                    aggregatedTasks.Add(taskEntryID);
                }
            }

            SatyamResultsAnalysis.RecordAggregationLog(noResultsNeededForAggregation_new, configString, guid);


            string outputmatFile = DirectoryConstants.defaultTempDirectory + guid + "\\" + configString + "_mat.txt";

            EvaluateAndPrintConfusionMatrixOfAggregatedResultEntries(aggEntries, outputmatFile, out noCorrect, prepareDataForTraining, outputDirectory);


            Console.WriteLine("noTotalConverged {0}", noTotalConverged);
            Console.WriteLine("noTerminatedTasks {0}", noTerminatedTasks);
            Console.WriteLine("Result: {0}/{1}, precision: {2}", noCorrect, noTotalConverged, (double)noCorrect / noTotalConverged);
            /// local file
            string outputString = String.Format("{0} {1} {2} {3} {4} {5}\n", configString, noCorrect, noTotalConverged, (double)noCorrect / noTotalConverged, noTerminatedTasks, ResultsPerTask.Count - noTotalConverged);
            string outputfile   = DirectoryConstants.defaultTempDirectory + guid + "\\resultSummary.txt";

            File.AppendAllText(outputfile, outputString);



            //for (double prob = 0; prob < 1;prob +=0.2)
            //{
            //    SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entries, guid, configString, anotherChanceProbablity: prob);
            //}
            //for (double ratio = 0; ratio < 1; ratio += 0.2)
            //{
            //    SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guid, configString, approvalRatioThreshold: ratio);
            //}
            SatyamResultsAnalysis.AggregationAnalysis(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guid, configString);
        }
        public static void EvaluateAndPrintConfusionMatrixOfAggregatedResultEntries(List <SatyamAggregatedResultsTableEntry> aggResultEntries,
                                                                                    string outputFile,
                                                                                    out int noCorrect,
                                                                                    //out SortedDictionary<string, Dictionary<string, int>> confusionMatrix_res_groundtruth,
                                                                                    //out SortedDictionary<string, Dictionary<string, int>> confusionMatrix_groundtruth_res,
                                                                                    bool prepareDataForTraining = false, string outputDirectory = null)
        {
            noCorrect = 0;
            //SortedDictionary<string, Dictionary<string, int>> confusionMatrix_res_groundtruth = new SortedDictionary<string, Dictionary<string, int>>();
            //SortedDictionary<string, Dictionary<string, int>> confusionMatrix_groundtruth_res = new SortedDictionary<string, Dictionary<string, int>>();

            List <KeyValuePair <string, string> > detections_gts = new List <KeyValuePair <string, string> >();


            WebClient wc = new WebClient();

            foreach (SatyamAggregatedResultsTableEntry aggResultEntry in aggResultEntries)
            {
                SatyamSaveAggregatedDataSatyam       data      = new SatyamSaveAggregatedDataSatyam(aggResultEntry);
                SingleObjectLabelingAggregatedResult aggResult = JSonUtils.ConvertJSonToObject <SingleObjectLabelingAggregatedResult>(data.AggregatedResultString);
                string fileName = URIUtilities.filenameFromURI(data.SatyamURI);
                string VideoCategoryGroundTruth = getVideoCategoryFromFileName(fileName);

                detections_gts.Add(new KeyValuePair <string, string>(aggResult.Category, VideoCategoryGroundTruth));

                //if (!confusionMatrix_res_groundtruth.ContainsKey(aggResult.Category))
                //{
                //    confusionMatrix_res_groundtruth.Add(aggResult.Category, new Dictionary<string, int>());
                //}
                //if (!confusionMatrix_res_groundtruth[aggResult.Category].ContainsKey(VideoCategoryGroundTruth))
                //{
                //    confusionMatrix_res_groundtruth[aggResult.Category].Add(VideoCategoryGroundTruth, 0);
                //}
                //if (!confusionMatrix_groundtruth_res.ContainsKey(VideoCategoryGroundTruth))
                //{
                //    confusionMatrix_groundtruth_res.Add(VideoCategoryGroundTruth, new Dictionary<string, int>());
                //}
                //if (!confusionMatrix_groundtruth_res[VideoCategoryGroundTruth].ContainsKey(aggResult.Category))
                //{
                //    confusionMatrix_groundtruth_res[VideoCategoryGroundTruth].Add(aggResult.Category, 0);
                //}
                //confusionMatrix_res_groundtruth[aggResult.Category][VideoCategoryGroundTruth]++;
                //confusionMatrix_groundtruth_res[VideoCategoryGroundTruth][aggResult.Category]++;

                if (aggResult.Category.Equals(VideoCategoryGroundTruth, StringComparison.InvariantCultureIgnoreCase))
                {
                    noCorrect++;
                }
                else
                {
                    Console.WriteLine("{0}, Groundtruth: {1}, Aggregated: {2}, Votes: {3}",
                                      fileName, VideoCategoryGroundTruth, aggResult.Category,
                                      JSonUtils.ConvertObjectToJSon(aggResult.metaData));
                }



                // prepare training dataset
                if (prepareDataForTraining)
                {
                    if (!Directory.Exists(outputDirectory + "\\" + aggResult.Category))
                    {
                        Directory.CreateDirectory(outputDirectory + "\\" + aggResult.Category);
                    }
                    wc.DownloadFile(data.SatyamURI, outputDirectory + "\\" + aggResult.Category + "\\" + fileName);
                }
            }


            SortedDictionary <string, Dictionary <string, int> > confusionMatrix_res_groundtruth;
            SortedDictionary <string, Dictionary <string, int> > confusionMatrix_groundtruth_res;

            SatyamResultValidationToolKit.getConfusionMat(detections_gts, out confusionMatrix_res_groundtruth, out confusionMatrix_groundtruth_res);
            SatyamResultValidationToolKit.printConfusionMatrix(confusionMatrix_groundtruth_res, confusionMatrix_res_groundtruth, outputFile);
        }
Beispiel #3
0
        public static void AggregateWithParameter(string guid,
                                                  int MinResults = TaskConstants.TRACKLET_LABELING_MTURK_MIN_RESULTS_TO_AGGREGATE,
                                                  int MaxResults = TaskConstants.TRACKLET_LABELING_MTURK_MAX_RESULTS_TO_AGGREGATE,
                                                  double boxToleranceThreshold = TaskConstants.TRACKLET_LABELING_BOX_DEVIATION_THRESHOLD,
                                                  double ObjectCoverageApprovalThresholdPerVideo = TaskConstants.TRACKLET_LABELING_APPROVALRATIO_PER_VIDEO,
                                                  double BoxCoverageApprovalThresholdPerTrack    = TaskConstants.TRACKLET_LABELING_APPROVALRATIO_PER_TRACK,
                                                  int consensusNumber = TaskConstants.TRACKLET_LABELING_MIN_RESULTS_FOR_CONSENSUS,
                                                  double minTubeletIoUSimilarityThreshold = TaskConstants.TRACKLET_LABELING_MIN_TUBELET_SIMILARITY_THRESHOLD,
                                                  double attributeMajority      = TaskConstants.TRACKLET_LABELING_MTURK_ATTRIBUTE_MAJORITY_THRESHOLD,
                                                  bool allFalseAttributeInvalid = false
                                                  )
        {
            string configString = "Min_" + MinResults + "_Max_" + MaxResults + "_IoU_" + minTubeletIoUSimilarityThreshold + "_Ratio_" + ObjectCoverageApprovalThresholdPerVideo;

            Console.WriteLine("Aggregating for " + guid + " with param set " + configString);
            SatyamResultsTableAccess       resultsDB = new SatyamResultsTableAccess();
            List <SatyamResultsTableEntry> entries   = resultsDB.getEntriesByGUID(guid);

            resultsDB.close();

            SortedDictionary <DateTime, List <SatyamResultsTableEntry> > entriesBySubmitTime = SatyamResultValidationToolKit.SortResultsBySubmitTime_OneResultPerTurkerPerTask(entries);

            Dictionary <int, List <MultiObjectTrackingResult> > ResultsPerTask = new Dictionary <int, List <MultiObjectTrackingResult> >();
            List <int> aggregatedTasks = new List <int>();

            int noTotalConverged = 0;
            //int noCorrect = 0;
            int noTerminatedTasks = 0;

            List <SatyamAggregatedResultsTableEntry> aggEntries = new List <SatyamAggregatedResultsTableEntry>();

            Dictionary <int, int> noResultsNeededForAggregation     = SatyamResultsAnalysis.getNoResultsNeededForAggregationFromLog(configString, guid);
            Dictionary <int, int> noResultsNeededForAggregation_new = new Dictionary <int, int>();
            // play back by time
            Dictionary <int, List <string> > WorkersPerTask = new Dictionary <int, List <string> >();

            foreach (DateTime t in entriesBySubmitTime.Keys)
            {
                //Console.WriteLine("Processing Results of time: {0}", t);
                List <SatyamResultsTableEntry> ResultEntries = entriesBySubmitTime[t];
                foreach (SatyamResultsTableEntry entry in ResultEntries)
                {
                    SatyamResult satyamResult           = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
                    SatyamTask   task                   = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
                    MultiObjectTrackingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(task.jobEntry.JobParameters);
                    string fileName    = URIUtilities.filenameFromURINoExtension(task.SatyamURI);
                    int    taskEntryID = entry.SatyamTaskTableEntryID;
                    if (aggregatedTasks.Contains(taskEntryID))
                    {
                        continue;
                    }
                    if (!ResultsPerTask.ContainsKey(taskEntryID))
                    {
                        ResultsPerTask.Add(taskEntryID, new List <MultiObjectTrackingResult>());
                        WorkersPerTask.Add(taskEntryID, new List <string>());
                    }

                    // remove duplicate workers result
                    string workerID = satyamResult.amazonInfo.WorkerID;
                    if (WorkersPerTask[taskEntryID].Contains(workerID))
                    {
                        continue;
                    }
                    //enclose only non-duplicate results, one per each worker.
                    WorkersPerTask[taskEntryID].Add(workerID);



                    string   videoName        = URIUtilities.localDirectoryNameFromURI(task.SatyamURI);
                    string[] fields           = videoName.Split('_');
                    int      startingFrame    = Convert.ToInt32(fields[fields.Length - 1]);
                    int      maxChunkEndFrame = startingFrame + job.ChunkDuration * job.FrameRate;
                    int      noFrameOverlap   = (int)(job.ChunkOverlap * job.FrameRate);
                    if (startingFrame != 0)
                    {
                        startingFrame -= noFrameOverlap;
                    }



                    string blobDir = URIUtilities.localDirectoryFullPathFromURI(task.SatyamURI);
                    VATIC_DVA_CrowdsourcedResult taskr = TrackletLabelingAggregator.createVATIC_DVA_CrowdsourcedResultUsingSatyamBlobImageCount(satyamResult.TaskResult, blobDir, entry.SatyamTaskTableEntryID.ToString(), job.FrameRate);
                    //MultiObjectTrackingResult result_tmp = vatic_tmp.getCompressedTracksInTimeSegment();

                    //VATIC_DVA_CrowdsourcedResult taskr = new VATIC_DVA_CrowdsourcedResult(satyamResult.TaskResult, videoName, entry.ID.ToString(), ImageURLs.Count, job.FrameRate);
                    MultiObjectTrackingResult res = taskr.getCompressedTracksInTimeSegment();

                    if (allFalseAttributeInvalid)
                    {
                        if (TrackletLabelingAggregator.AllAttributeAllFalse(res))
                        {
                            continue;
                        }
                    }

                    ResultsPerTask[taskEntryID].Add(res);

                    // check log if enough results are collected

                    if (noResultsNeededForAggregation != null && noResultsNeededForAggregation.ContainsKey(taskEntryID) &&
                        ResultsPerTask[taskEntryID].Count < noResultsNeededForAggregation[taskEntryID])
                    {
                        continue;
                    }



                    // hack for masters
                    int tempMin = MinResults;
                    if (TaskConstants.masterGUIDs.Contains(entry.JobGUID))
                    {
                        tempMin = 1;
                    }

                    TrackletLabelingAggregatedResult aggResult = TrackletLabelingAggregator.getAggregatedResult(ResultsPerTask[taskEntryID],
                                                                                                                guid, task.SatyamURI,
                                                                                                                job.FrameRate, job.BoundaryLines,
                                                                                                                MinResults, MaxResults, boxToleranceThreshold,
                                                                                                                ObjectCoverageApprovalThresholdPerVideo,
                                                                                                                BoxCoverageApprovalThresholdPerTrack,
                                                                                                                consensusNumber, minTubeletIoUSimilarityThreshold,
                                                                                                                attributeMajority);

                    if (aggResult == null)
                    {
                        continue;
                    }

                    //////////////// aggregation happen
                    // record logs
                    if (noResultsNeededForAggregation == null || !noResultsNeededForAggregation.ContainsKey(taskEntryID))
                    {
                        noResultsNeededForAggregation_new.Add(taskEntryID, ResultsPerTask[taskEntryID].Count);
                    }



                    aggregatedTasks.Add(taskEntryID);
                    noTotalConverged++;
                    if (ResultsPerTask[taskEntryID].Count >= MaxResults)
                    {
                        noTerminatedTasks++;
                    }
                    SatyamAggregatedResult SatyamAggResult = new SatyamAggregatedResult();
                    SatyamAggResult.SatyamTaskTableEntryID = taskEntryID;
                    SatyamAggResult.AggregatedResultString = JSonUtils.ConvertObjectToJSon <TrackletLabelingAggregatedResult>(aggResult);
                    SatyamAggResult.TaskParameters         = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString).TaskParametersString;

                    SatyamAggregatedResultsTableEntry aggEntry = new SatyamAggregatedResultsTableEntry();
                    aggEntry.SatyamTaskTableEntryID = taskEntryID;
                    aggEntry.JobGUID      = entry.JobGUID;
                    aggEntry.ResultString = JSonUtils.ConvertObjectToJSon <SatyamAggregatedResult>(SatyamAggResult);


                    aggEntries.Add(aggEntry);
                }
            }


            Console.WriteLine("Total_Aggregated_Tasks: {0}", noTotalConverged);
            Console.WriteLine("Total_Terminated_Tasks: {0}", noTerminatedTasks);

            SatyamResultsAnalysis.RecordAggregationLog(noResultsNeededForAggregation_new, configString, guid);

            TrackletLabelingAnalyzer.GroupEntriesByVideoNameAndStitchAndSaveAggregatedResultVideosLocally(aggEntries, DirectoryConstants.defaultTempDirectory + guid);
        }
Beispiel #4
0
        public static void AggregateWithParameterAndValidateObjectCountingResultByGUID(string guid,
                                                                                       //bool saveImage = false,
                                                                                       bool prepareTrainingSet = false, string outputDirectory = null,
                                                                                       int MinHeight           = TaskConstants.OBJECT_COUNTING_VALIDATION_MIN_HEIGHT,
                                                                                       int MaxOcclusion        = TaskConstants.OBJECT_COUNTING_VALIDATION_MAX_OCCLUSION,
                                                                                       double Max_Truncation   = TaskConstants.OBJECT_COUNTING_VALIDATION_MIN_TRUNCATION,
                                                                                       int MinResults          = TaskConstants.OBJECT_COUNTING_MTURK_MIN_RESULTS_TO_AGGREGATE,
                                                                                       int MaxResults          = TaskConstants.OBJECT_COUNTING_MTURK_MAX_RESULTS_TO_AGGREGATE,
                                                                                       double MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND = TaskConstants.OBJECT_COUNTING_MTURK_MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND,
                                                                                       double MAX_DEVIATION_FRACTION = TaskConstants.OBJECT_COUNTING_MTURK_MAX_DEVIATION_FRACTION,
                                                                                       double SUPER_MAJORITY_VALUE   = TaskConstants.OBJECT_COUNTING_MTURK_SUPER_MAJORITY_VALUE
                                                                                       )
        {
            if (!Directory.Exists(DirectoryConstants.defaultTempDirectory + guid))
            {
                Directory.CreateDirectory(DirectoryConstants.defaultTempDirectory + guid);
            }

            string configString = "Min_" + MinResults + "_Max_" + MaxResults + "_Dev_" + MAX_DEVIATION_FRACTION + "_Majority_" + SUPER_MAJORITY_VALUE;

            Console.WriteLine("Aggregating for param set " + configString);
            int noTerminatedTasks = 0;
            SatyamResultsTableAccess       resultsDB = new SatyamResultsTableAccess();
            List <SatyamResultsTableEntry> entries   = resultsDB.getEntriesByGUIDOrderByID(guid);

            resultsDB.close();

            SortedDictionary <DateTime, List <SatyamResultsTableEntry> > entriesBySubmitTime = SatyamResultValidationToolKit.SortResultsBySubmitTime(entries);

            int noTotalConverged = 0;
            //int noCorrect = 0;

            Dictionary <int, List <SatyamResultsTableEntry> > ResultsPerTask = new Dictionary <int, List <SatyamResultsTableEntry> >();
            List <int> aggregatedTasks = new List <int>();

            List <SatyamAggregatedResultsTableEntry> aggEntries     = new List <SatyamAggregatedResultsTableEntry>();
            Dictionary <int, int> noResultsNeededForAggregation     = SatyamResultsAnalysis.getNoResultsNeededForAggregationFromLog(configString, guid);
            Dictionary <int, int> noResultsNeededForAggregation_new = new Dictionary <int, int>();

            // play back by time
            foreach (DateTime t in entriesBySubmitTime.Keys)
            {
                //Console.WriteLine("Processing Results of time: {0}", t);
                foreach (SatyamResultsTableEntry entry in entriesBySubmitTime[t])
                {
                    SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
                    SatyamTask   task         = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
                    SatyamJob    job          = task.jobEntry;
                    string       fileName     = URIUtilities.filenameFromURI(task.SatyamURI);


                    int taskEntryID = entry.SatyamTaskTableEntryID;
                    if (aggregatedTasks.Contains(taskEntryID))
                    {
                        continue;
                    }

                    if (!ResultsPerTask.ContainsKey(taskEntryID))
                    {
                        ResultsPerTask.Add(taskEntryID, new List <SatyamResultsTableEntry>());
                    }
                    ResultsPerTask[taskEntryID].Add(entry);

                    // check log if enough results are collected
                    if (noResultsNeededForAggregation != null && noResultsNeededForAggregation.ContainsKey(taskEntryID) &&
                        ResultsPerTask[taskEntryID].Count < noResultsNeededForAggregation[taskEntryID])
                    {
                        continue;
                    }

                    string aggResultString = ObjectCountingAggregator.GetAggregatedResultString(ResultsPerTask[taskEntryID], MinResults, MaxResults, MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND, MAX_DEVIATION_FRACTION, SUPER_MAJORITY_VALUE);

                    if (aggResultString == null)
                    {
                        continue;
                    }

                    SatyamAggregatedResultsTableEntry aggEntry = new SatyamAggregatedResultsTableEntry();
                    aggEntry.JobGUID                = ResultsPerTask[taskEntryID][0].JobGUID;
                    aggEntry.JobTemplateType        = ResultsPerTask[taskEntryID][0].JobTemplateType;
                    aggEntry.SatyamTaskTableEntryID = taskEntryID;
                    aggEntry.UserID       = ResultsPerTask[taskEntryID][0].UserID;
                    aggEntry.ResultString = aggResultString;
                    /// aggregation happen
                    /// // record logs
                    // record logs
                    if (noResultsNeededForAggregation == null || !noResultsNeededForAggregation.ContainsKey(taskEntryID))
                    {
                        noResultsNeededForAggregation_new.Add(taskEntryID, ResultsPerTask[taskEntryID].Count);
                    }
                    aggEntries.Add(aggEntry);
                    noTotalConverged++;
                    if (ResultsPerTask[taskEntryID].Count >= MaxResults)
                    {
                        noTerminatedTasks++;
                    }
                    aggregatedTasks.Add(taskEntryID);
                }
            }

            SatyamResultsAnalysis.RecordAggregationLog(noResultsNeededForAggregation_new, configString, guid);

            double totalError, totalGroundtruth;

            //ValidateSatyamKITTIObjectCountingAggregationResult(aggEntries, out totalError, out totalGroundtruth,
            //    MinHeight, MaxOcclusion, Max_Truncation);
            ValidateSatyamCARPKObjectCountingAggregationResult(aggEntries, out totalError, out totalGroundtruth);

            Console.WriteLine("noTotalConverged {0}", noTotalConverged);
            Console.WriteLine("noTerminatedTasks {0}", noTerminatedTasks);
            /// local file
            string outputString = String.Format("{0} {1} {2} {3} {4} {5} {6}\n", configString, totalError, totalGroundtruth, totalError / totalGroundtruth, noTotalConverged, noTerminatedTasks, ResultsPerTask.Count - noTotalConverged);
            string outputfile   = DirectoryConstants.defaultTempDirectory + guid + "\\resultSummary.txt";

            File.AppendAllText(outputfile, outputString);

            string approvalString = configString + "_PayDev_" + TaskConstants.OBJECT_COUNTING_MTURK_MAX_ABSOLUTE_COUNT_DEVIATION_FOR_PAYMENT +
                                    "_PayFrac_" + TaskConstants.OBJECT_COUNTING_MTURK_MAX_DEVIATION_FRACTION_FOR_PAYMENT;

            //for (double prob = 0; prob < 1;prob +=0.2)
            //{
            //    SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entries, guid, approvalString, anotherChanceProbablity: prob);
            //}
            //for (double ratio = 0; ratio < 1; ratio += 0.2)
            //{
            //    SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guid, configString, approvalRatioThreshold: ratio);
            //}
            SatyamResultsAnalysis.AggregationAnalysis(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guid, configString);
        }
Beispiel #5
0
        public static void validateImageNet1000ClassDetectionResult()
        {
            string baseDir = @"C:\research\dataset\inception1000\";
            //string classNameFile = baseDir + "1000class.txt";
            string resultFile      = baseDir + "results_inception_Seattle.txt";
            string is_a_File       = baseDir + "wordnet.is_a.txt";
            string wnid_words_File = baseDir + "words.txt";
            string synsetFile      = baseDir + "synset.txt";

            //List<string> classNameContent = File.ReadAllLines(classNameFile).ToList();

            //Dictionary<int, string> classNames = new Dictionary<int, string>();
            //for (int i=0;i<classNameContent.Count;i++)
            //{
            //    string[] fields = classNameContent[i].Split(':');
            //    string[] names = fields[1].Split('\'');
            //    string name = "";
            //    for(int j = 1; j < names.Length - 1; j++)
            //    {
            //        name += names[j];
            //        if (j != names.Length - 2)
            //        {
            //            name += '\'';
            //        }
            //    }
            //    classNames.Add(i, name);
            //}

            List <string> synsetContent = File.ReadAllLines(synsetFile).ToList();

            Dictionary <int, string> synsetNames = new Dictionary <int, string>();

            for (int i = 0; i < synsetContent.Count; i++)
            {
                string[] fields = synsetContent[i].Split(' ');
                synsetNames.Add(i, fields[0]);
            }


            List <string> is_a_Content = File.ReadLines(is_a_File).ToList();
            Dictionary <string, List <string> > is_a = new Dictionary <string, List <string> >();

            foreach (string isA in is_a_Content)
            {
                string[] fields = isA.Split(' ');
                if (!is_a.ContainsKey(fields[1]))
                {
                    is_a.Add(fields[1], new List <string>());
                }
                is_a[fields[1]].Add(fields[0]);
            }

            List <string> wordsContent = File.ReadAllLines(wnid_words_File).ToList();
            Dictionary <string, List <string> > word_wnid = new Dictionary <string, List <string> >();

            foreach (string word in wordsContent)
            {
                string[] fields = word.Split('\t');
                //word_wnid.Add(fields[1], fields[0]);
                if (!word_wnid.ContainsKey(fields[1]))
                {
                    word_wnid.Add(fields[1], new List <string>());
                }
                word_wnid[fields[1]].Add(fields[0]);
            }

            List <string> resContent = File.ReadAllLines(resultFile).ToList();
            List <KeyValuePair <double, int> > results = new List <KeyValuePair <double, int> >();
            List <int> gt = new List <int>();

            foreach (string res in resContent)
            {
                string[] fields     = res.Split('[');
                double   confidence = Convert.ToDouble(fields[1].Split(']')[0]);
                int      classIdx   = Convert.ToInt32(fields[2].Split(']')[0]);
                results.Add(new KeyValuePair <double, int>(confidence, classIdx));
                gt.Add(Convert.ToInt32(fields[2].Split(' ')[1]));
            }

            List <string> result_wnid     = new List <string>();
            List <string> result_category = new List <string>();


            List <string> dets_gts_oneString = new List <string>();

            List <string> categoriesFound = new List <string>();

            for (int i = 0; i < results.Count; i++)
            {
                KeyValuePair <double, int> res = results[i];

                if (res.Value == 0)
                {
                    result_wnid.Add("");
                    result_category.Add("");
                    continue;
                }
                string synsetName = synsetNames[res.Value - 1];


                //string classwnid = word_wnid[classname];
                List <string> is_wnid = new List <string>();
                is_wnid.Add(synsetName);
                //foreach(string classwnid in word_wnid[synsetName])
                //{
                //    is_wnid.Add(classwnid);
                //}

                for (int j = 0; j < is_wnid.Count; j++)
                {
                    if (!is_a.ContainsKey(is_wnid[j]))
                    {
                        continue;
                    }
                    foreach (string wnid in is_a[is_wnid[j]])
                    {
                        is_wnid.Add(wnid);
                    }
                }

                bool found = false;
                foreach (string wnid in is_wnid)
                {
                    if (GroundTruth.ContainsKey(wnid))
                    {
                        result_wnid.Add(wnid);
                        result_category.Add(GroundTruth[wnid]);
                        found = true;
                        if (!categoriesFound.Contains(GroundTruth[wnid]))
                        {
                            categoriesFound.Add(GroundTruth[wnid]);
                        }
                        break;
                    }
                }
                if (!found)
                {
                    result_wnid.Add("");
                    result_category.Add("");
                }

                string gtString = GroundTruth_AlphabeticalOrder[gt[i] - 1];
                dets_gts_oneString.Add(synsetName + " " + gtString);
            }

            int noCorrect          = 0;
            int noTotal            = results.Count;
            int noTotalPredictions = 0;

            List <KeyValuePair <string, string> > dets_gts = new List <KeyValuePair <string, string> >();


            for (int i = 0; i < results.Count; i++)
            {
                string detection = result_category[i];
                string gtString  = GroundTruth_AlphabeticalOrder[gt[i] - 1];
                dets_gts.Add(new KeyValuePair <string, string>(detection, gtString));

                if (result_category[i] != "" && result_category[i] != "amulance")
                {
                    noTotalPredictions++;

                    if (detection.Equals(gtString, StringComparison.InvariantCultureIgnoreCase))
                    {
                        noCorrect++;
                    }
                    else
                    {
                        Console.WriteLine(detection + " " + gtString);
                    }
                }
            }

            SortedDictionary <string, Dictionary <string, int> > confusionMatrix_res_groundtruth;
            SortedDictionary <string, Dictionary <string, int> > confusionMatrix_groundtruth_res;

            SatyamResultValidationToolKit.getConfusionMat(dets_gts, out confusionMatrix_res_groundtruth, out confusionMatrix_groundtruth_res);
            string outputFile = @"C:\research\dataset\inception1000\confusionMat.txt";

            SatyamResultValidationToolKit.printConfusionMatrix(confusionMatrix_groundtruth_res, confusionMatrix_res_groundtruth, outputFile);

            outputFile = @"C:\research\dataset\inception1000\det_gts.txt";
            File.WriteAllLines(outputFile, dets_gts_oneString.ToArray());
        }
Beispiel #6
0
        public static void AggregateWithParameterAndValidateSatyamImageNetClassificationResultByGUID(string guid,
                                                                                                     int MinResults                    = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MIN_RESULTS_TO_AGGREGATE,
                                                                                                     int MaxResults                    = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MAX_RESULTS_TO_AGGREGATE,
                                                                                                     double MajorityThreshold          = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MAJORITY_THRESHOLD,
                                                                                                     string confusingImageListFilePath = null)
        {
            string configString = "Min_" + MinResults + "_Max_" + MaxResults + "_Thresh_" + MajorityThreshold;

            Console.WriteLine("Aggregating for param set " + configString);
            int noTerminatedTasks = 0;

            SatyamResultsTableAccess       resultsDB = new SatyamResultsTableAccess();
            List <SatyamResultsTableEntry> entries   = resultsDB.getEntriesByGUIDOrderByID(guid);

            resultsDB.close();

            //SortedDictionary<DateTime, List<SatyamResultsTableEntry>> entriesBySubmitTime =
            //    SatyamResultValidation.SortResultsBySubmitTime_OneResultPerTurkerPerTask(entries);
            SortedDictionary <DateTime, List <SatyamResultsTableEntry> > entriesBySubmitTime =
                SatyamResultValidationToolKit.SortResultsBySubmitTime(entries);

            Dictionary <string, ConfusingReason> imageBlackListReason = new Dictionary <string, ConfusingReason>();

            if (confusingImageListFilePath != null)
            {
                imageBlackListReason = getConfusingImageList(confusingImageListFilePath);
            }

            //Dictionary<int, List<SingleObjectLabelingResult>> ResultsPerTask = new Dictionary<int, List<SingleObjectLabelingResult>>();
            Dictionary <int, List <SatyamResultsTableEntry> > ResultsPerTask = new Dictionary <int, List <SatyamResultsTableEntry> >();

            List <int> aggregatedTasks = new List <int>();

            int noTotalConverged = 0;
            int noCorrect        = 0;

            SortedDictionary <string, Dictionary <string, int> > confusionMatrix_res_groundtruth = new SortedDictionary <string, Dictionary <string, int> >();
            SortedDictionary <string, Dictionary <string, int> > confusionMatrix_groundtruth_res = new SortedDictionary <string, Dictionary <string, int> >();

            StringBuilder s = new StringBuilder();

            s.Append("<!DOCTYPE html>\n");
            s.Append("<html>\n");
            s.Append("<body>\n");
            String title = String.Format("<h1>Job GUID {0} Incorrect Result Summary</h1>\n", guid);

            s.Append(title);

            Dictionary <int, int> noResultsNeededForAggregation     = SatyamResultsAnalysis.getNoResultsNeededForAggregationFromLog(configString, guid);
            Dictionary <int, int> noResultsNeededForAggregation_new = new Dictionary <int, int>();

            List <SatyamAggregatedResultsTableEntry> aggEntries = new List <SatyamAggregatedResultsTableEntry>();

            foreach (DateTime t in entriesBySubmitTime.Keys)
            {
                foreach (SatyamResultsTableEntry entry in entriesBySubmitTime[t])
                {
                    //SatyamResultsTableEntry entry = entries[i];
                    SatyamResult satyamResult      = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
                    SatyamTask   task              = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
                    SatyamJob    job               = task.jobEntry;
                    string       fileName          = URIUtilities.filenameFromURI(task.SatyamURI);
                    string       imageCategoryName = fileName.Split('_')[0];

                    if (IsBlackListed(task.SatyamURI, imageBlackListReason))
                    {
                        continue;
                    }
                    int taskEntryID = entry.SatyamTaskTableEntryID;
                    if (aggregatedTasks.Contains(taskEntryID))
                    {
                        continue;
                    }

                    if (!ResultsPerTask.ContainsKey(taskEntryID))
                    {
                        ResultsPerTask.Add(taskEntryID, new List <SatyamResultsTableEntry>());
                    }
                    //ResultEntriesPerTask[taskEntryID].Add(JSonUtils.ConvertJSonToObject<SingleObjectLabelingResult>(satyamResult.TaskResult));
                    ResultsPerTask[taskEntryID].Add(entry);

                    // check log if enough results are collected
                    if (noResultsNeededForAggregation != null && noResultsNeededForAggregation.ContainsKey(taskEntryID) &&
                        ResultsPerTask[taskEntryID].Count < noResultsNeededForAggregation[taskEntryID])
                    {
                        continue;
                    }


                    string aggResultString = SingleObjectLabelingAggregator.GetAggregatedResultString(ResultsPerTask[taskEntryID], MinResults, MaxResults, MajorityThreshold);
                    //SingleObjectLabelingAggregatedResult aggResult = SingleObjectLabelingAggregator.getAggregatedResult(ResultEntriesPerTask[taskEntryID], MinResults, MaxResults, MajorityThreshold);


                    if (aggResultString == null)
                    {
                        continue;
                    }



                    SatyamAggregatedResultsTableEntry aggEntry = new SatyamAggregatedResultsTableEntry();
                    aggEntry.JobGUID                = ResultsPerTask[taskEntryID][0].JobGUID;
                    aggEntry.JobTemplateType        = ResultsPerTask[taskEntryID][0].JobTemplateType;
                    aggEntry.SatyamTaskTableEntryID = ResultsPerTask[taskEntryID][0].SatyamTaskTableEntryID;
                    aggEntry.UserID       = ResultsPerTask[taskEntryID][0].UserID;
                    aggEntry.ResultString = aggResultString;

                    /// aggregation happen
                    // record logs
                    if (noResultsNeededForAggregation == null || !noResultsNeededForAggregation.ContainsKey(taskEntryID))
                    {
                        noResultsNeededForAggregation_new.Add(taskEntryID, ResultsPerTask[taskEntryID].Count);
                    }
                    ///
                    //if (aggResult.Category == "None of the Above")
                    //{
                    //    continue;
                    //}

                    aggEntries.Add(aggEntry);

                    SatyamSaveAggregatedDataSatyam data = new SatyamSaveAggregatedDataSatyam(aggEntry);
                    String resultString = data.AggregatedResultString;
                    SingleObjectLabelingAggregatedResult aggResult = JSonUtils.ConvertJSonToObject <SingleObjectLabelingAggregatedResult>(resultString);

                    if (!confusionMatrix_res_groundtruth.ContainsKey(aggResult.Category))
                    {
                        confusionMatrix_res_groundtruth.Add(aggResult.Category, new Dictionary <string, int>()
                        {
                            { GroundTruth[imageCategoryName], 0 }
                        });
                    }
                    else
                    {
                        if (!confusionMatrix_res_groundtruth[aggResult.Category].ContainsKey(GroundTruth[imageCategoryName]))
                        {
                            confusionMatrix_res_groundtruth[aggResult.Category].Add(GroundTruth[imageCategoryName], 0);
                        }
                    }

                    if (!confusionMatrix_groundtruth_res.ContainsKey(GroundTruth[imageCategoryName]))
                    {
                        confusionMatrix_groundtruth_res.Add(GroundTruth[imageCategoryName], new Dictionary <string, int>()
                        {
                            { aggResult.Category, 0 }
                        });
                    }
                    else
                    {
                        if (!confusionMatrix_groundtruth_res[GroundTruth[imageCategoryName]].ContainsKey(aggResult.Category))
                        {
                            confusionMatrix_groundtruth_res[GroundTruth[imageCategoryName]].Add(aggResult.Category, 0);
                        }
                    }
                    if (aggResult.Category.Equals(GroundTruth[imageCategoryName], StringComparison.InvariantCultureIgnoreCase))
                    {
                        noCorrect++;
                    }
                    else
                    {
                        //Console.WriteLine("{0}, Groundtruth: {1}, Aggregated: {2}, Votes: {3}",
                        //    fileName, GroundTruth[imageCategoryName], aggResult.Category,
                        //    JSonUtils.ConvertObjectToJSon(aggResult.metaData));

                        String record = String.Format("<p>{0}, Groundtruth: {1}, Aggregated: {2}, Votes: {3}</p>\n",
                                                      fileName, GroundTruth[imageCategoryName], aggResult.Category,
                                                      JSonUtils.ConvertObjectToJSon(aggResult.metaData));
                        String img = String.Format("<img src=\"{0}\" >\n", task.SatyamURI);
                        s.Append(record);
                        s.Append(img);
                    }
                    noTotalConverged++;
                    if (ResultsPerTask[taskEntryID].Count >= MaxResults)
                    {
                        noTerminatedTasks++;
                    }
                    confusionMatrix_res_groundtruth[aggResult.Category][GroundTruth[imageCategoryName]]++;
                    confusionMatrix_groundtruth_res[GroundTruth[imageCategoryName]][aggResult.Category]++;
                    aggregatedTasks.Add(taskEntryID);
                }
            }

            SatyamResultsAnalysis.RecordAggregationLog(noResultsNeededForAggregation_new, configString, guid);

            s.Append("</body>\n");
            s.Append("</html>\n");
            string dataToBeSaved = s.ToString();

            SatyamJobStorageAccountAccess storage = new SatyamJobStorageAccountAccess();
            string FileName = String.Format("AggregatedIncorrectResults-{0}_Min{1}Max{2}Thresh{3}.html", guid, MinResults, MaxResults, MajorityThreshold);

            storage.SaveATextFile("singleobjectlabeling", guid, FileName, dataToBeSaved);

            s.Clear();
            s.Append("<!DOCTYPE html>\n");
            s.Append("<html>\n");
            s.Append("<body>\n");
            s.Append(title);

            string resultSummary = String.Format("<p>Result: {0}/{1}, precision: {2}</p>\n", noCorrect, noTotalConverged, (double)noCorrect / noTotalConverged);

            resultSummary += String.Format("<p>Terminated: {0}, Not Enough Results: {1}</p>\n", noTerminatedTasks, ResultsPerTask.Count - noTotalConverged);
            Console.WriteLine(resultSummary);
            s.Append(resultSummary);

            // write the confusion matrix
            s.Append("<p>");
            String row = "\t\t";

            foreach (string resultCategory in confusionMatrix_res_groundtruth.Keys)
            {
                row += resultCategory + "\t";
            }
            row += "<br>\n";
            s.Append(row);
            Console.WriteLine(row);

            string matString = "";

            foreach (string groundTruthCategory in confusionMatrix_groundtruth_res.Keys)
            {
                row = groundTruthCategory + "\t";
                foreach (string resultCategory in confusionMatrix_res_groundtruth.Keys)
                {
                    if (confusionMatrix_groundtruth_res[groundTruthCategory].ContainsKey(resultCategory))
                    {
                        row += confusionMatrix_groundtruth_res[groundTruthCategory][resultCategory].ToString();
                    }
                    else
                    {
                        row += "0";
                    }
                    row += "\t";
                }
                row += "<br>\n";
                s.Append(row);
                Console.WriteLine(row);
                matString += row;
            }

            s.Append("</p>\n");

            s.Append("</body>\n");
            s.Append("</html>\n");

            string summaryToBeSaved = s.ToString();

            FileName = String.Format("Aggregated_Summary-{0}_Min{1}Max{2}Thresh{3}.html", guid, MinResults, MaxResults, MajorityThreshold);
            storage.SaveATextFile("singleobjectlabeling", guid, FileName, summaryToBeSaved);

            /// local file
            string outputString = String.Format("{0} {1} {2} {3} {4} {5}\n", configString, noCorrect, noTotalConverged, (double)noCorrect / noTotalConverged, noTerminatedTasks, ResultsPerTask.Count - noTotalConverged);
            string outputfile   = DirectoryConstants.defaultTempDirectory + guid + "\\resultSummary.txt";

            File.AppendAllText(outputfile, outputString);
            string outputmatFile = DirectoryConstants.defaultTempDirectory + guid + "\\" + configString + "_mat.txt";

            File.WriteAllText(outputmatFile, matString);



            //for (double prob = 0; prob < 1;prob +=0.2)
            //{
            //    SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entries, guid, configString, anotherChanceProbablity: prob);
            //}
            //for (double ratio = 0; ratio < 1; ratio += 0.2)
            //{
            //    SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guid, configString, approvalRatioThreshold: ratio);
            //}
            SatyamResultsAnalysis.AggregationAnalysis(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guid, configString);
        }
        /// <summary>
        /// function generalizable.. TODO... to all tasks
        /// </summary>
        /// <param name="guids"></param>
        /// <param name="IoUTreshold"></param>
        /// <param name="saveImage"></param>
        /// <param name="outputDirectory"></param>
        /// <param name="MinResults"></param>
        /// <param name="MaxResults"></param>
        /// <param name="CategoryMajorityThreshold"></param>
        /// <param name="PolygonBoundaryMajorityThreshold"></param>
        /// <param name="ObjectsCoverageThreshold"></param>
        /// <param name="overwrite"></param>
        /// <param name="approvalAnalysis"></param>
        public static void AggregateWithParameterAndValidatePascalVOCImageSegmentationByGUID(List <string> guids,
                                                                                             double IoUTreshold,
                                                                                             bool saveImage = false, string outputDirectory = null,
                                                                                             int MinResults = TaskConstants.IMAGE_SEGMENTATION_MTURK_MIN_RESULTS_TO_AGGREGATE,
                                                                                             int MaxResults = TaskConstants.IMAGE_SEGMENTATION_MTURK_MAX_RESULTS_TO_AGGREGATE,
                                                                                             double CategoryMajorityThreshold        = TaskConstants.IMAGE_SEGMENTATION_MTURK_MAJORITY_CATEGORY_THRESHOLD,
                                                                                             double PolygonBoundaryMajorityThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_MAJORITY_POLYGON_BOUNDARY_THRESHOLD,
                                                                                             double ObjectsCoverageThreshold         = TaskConstants.IMAGE_SEGMENTATION_MTURK_OBJECT_COVERAGE_THRESHOLD_FOR_AGGREGATION_TERMINATION,
                                                                                             double minSimilarityThreshold           = TaskConstants.IMAGE_SEGMENTATION_MTURK_POLYGON_IOU_THRESHOLD,
                                                                                             int minResultsForConsensus = TaskConstants.IMAGE_SEGMENTATION_MTURK_MIN_RESULTS_FOR_CONSENSUS,
                                                                                             bool overwrite             = false,
                                                                                             bool approvalAnalysis      = false)
        {
            string configString = "Min_" + MinResults + "_Max_" + MaxResults + "_Majority_" + PolygonBoundaryMajorityThreshold + "_Ratio_" + ObjectsCoverageThreshold;

            Console.WriteLine("Aggregating for param set " + configString);
            if (!overwrite && File.Exists(DirectoryConstants.defaultTempDirectory + "\\ImageSegmentationResult\\" + configString + ".txt"))
            {
                return;
            }

            SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();

            Dictionary <int, List <string> > WorkersPerTask = new Dictionary <int, List <string> >();

            List <SatyamResultsTableEntry> entries = new List <SatyamResultsTableEntry>();

            foreach (string guid in guids)
            {
                entries.AddRange(resultsDB.getEntriesByGUIDOrderByID(guid));
            }
            resultsDB.close();

            SortedDictionary <DateTime, List <SatyamResultsTableEntry> > entriesBySubmitTime =
                SatyamResultValidationToolKit.SortResultsBySubmitTime_OneResultPerTurkerPerTask(entries);

            Dictionary <int, List <ImageSegmentationResult> > ResultsPerTask = new Dictionary <int, List <ImageSegmentationResult> >();
            List <int> aggregatedTasks = new List <int>();

            int noTotalConverged = 0;
            //int noCorrect = 0;
            int noTerminatedTasks = 0;

            List <SatyamAggregatedResultsTableEntry> aggEntries     = new List <SatyamAggregatedResultsTableEntry>();
            Dictionary <int, int> noResultsNeededForAggregation     = SatyamResultsAnalysis.getNoResultsNeededForAggregationFromLog(configString, guids[0]);
            Dictionary <int, int> noResultsNeededForAggregation_new = new Dictionary <int, int>();

            // play back by time
            foreach (DateTime t in entriesBySubmitTime.Keys)
            {
                //Console.WriteLine("Processing Results of time: {0}", t);
                List <SatyamResultsTableEntry> ResultEntries = entriesBySubmitTime[t];
                foreach (SatyamResultsTableEntry entry in ResultEntries)
                {
                    SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
                    SatyamTask   task         = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
                    SatyamJob    job          = task.jobEntry;
                    string       fileName     = URIUtilities.filenameFromURINoExtension(task.SatyamURI);
                    int          taskEntryID  = entry.SatyamTaskTableEntryID;
                    if (aggregatedTasks.Contains(taskEntryID))
                    {
                        continue;
                    }
                    if (!ResultsPerTask.ContainsKey(taskEntryID))
                    {
                        ResultsPerTask.Add(taskEntryID, new List <ImageSegmentationResult>());
                    }

                    ResultsPerTask[taskEntryID].Add(JSonUtils.ConvertJSonToObject <ImageSegmentationResult>(satyamResult.TaskResult));

                    // check log if enough results are collected
                    if (noResultsNeededForAggregation != null && noResultsNeededForAggregation.ContainsKey(taskEntryID) &&
                        ResultsPerTask[taskEntryID].Count < noResultsNeededForAggregation[taskEntryID])
                    {
                        continue;
                    }

                    ImageSegmentationAggregatedResult aggResult = ImageSegmentationAggregator.getAggregatedResult(
                        ResultsPerTask[taskEntryID], task.SatyamURI, entry.JobGUID,
                        MinResults, MaxResults, CategoryMajorityThreshold,
                        PolygonBoundaryMajorityThreshold, ObjectsCoverageThreshold,
                        minSimilarityThreshold, minResultsForConsensus);
                    if (aggResult == null)
                    {
                        continue;
                    }

                    // aggregation happen
                    // record logs
                    if (noResultsNeededForAggregation == null || !noResultsNeededForAggregation.ContainsKey(taskEntryID))
                    {
                        noResultsNeededForAggregation_new.Add(taskEntryID, ResultsPerTask[taskEntryID].Count);
                    }
                    aggregatedTasks.Add(taskEntryID);
                    noTotalConverged++;
                    if (ResultsPerTask[taskEntryID].Count >= MaxResults)
                    {
                        noTerminatedTasks++;
                    }
                    SatyamAggregatedResult SatyamAggResult = new SatyamAggregatedResult();
                    SatyamAggResult.SatyamTaskTableEntryID = taskEntryID;
                    SatyamAggResult.AggregatedResultString = JSonUtils.ConvertObjectToJSon <ImageSegmentationAggregatedResult>(aggResult);
                    SatyamAggResult.TaskParameters         = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString).TaskParametersString;

                    SatyamAggregatedResultsTableEntry aggEntry = new SatyamAggregatedResultsTableEntry();
                    aggEntry.SatyamTaskTableEntryID = taskEntryID;
                    aggEntry.JobGUID      = entry.JobGUID;
                    aggEntry.ResultString = JSonUtils.ConvertObjectToJSon <SatyamAggregatedResult>(SatyamAggResult);


                    aggEntries.Add(aggEntry);
                    List <SatyamAggregatedResultsTableEntry> tmpEntries = new List <SatyamAggregatedResultsTableEntry>();
                    tmpEntries.Add(aggEntry);

                    //ValidateSatyamKITTIDetectionAggregationResult(tmpEntries, saveImage, MinHeight, MaxOcclusion, Max_Truncation);
                }
            }
            Console.WriteLine("Total_Aggregated_Tasks: {0}", noTotalConverged);
            Console.WriteLine("Total_Terminated_Tasks: {0}", noTerminatedTasks);

            SatyamResultsAnalysis.RecordAggregationLog(noResultsNeededForAggregation_new, configString, guids[0]);

            string r = ValidatePascalVOCImageSegmentationResult_InstanceLevel(aggEntries, IoUTreshold);

            r = noTotalConverged + " " + noTerminatedTasks + " " + r;
            File.WriteAllText(DirectoryConstants.defaultTempDirectory + "\\ImageSegmentationResult\\" + configString + ".txt", r);


            if (approvalAnalysis)
            {
                string approvalString = configString + "_PayCover_" + TaskConstants.IMAGE_SEGMENTATION_MTURK_OBJECT_COVERAGE_THRESHOLD_FOR_PAYMENT +
                                        "_PayIoU_" + TaskConstants.IMAGE_SEGMENTATION_MTURK_POLYGON_IOU_THRESHOLD_FOR_PAYMENT;
                //for (double ratio = 0; ratio < 1; ratio += 0.2)
                //{
                //    SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guids[0], configString, approvalRatioThreshold: ratio);
                //}
                SatyamResultsAnalysis.AggregationAnalysis(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guids[0], configString);
            }
        }