Exemplo n.º 1
0
        public static void TestAggregationAnalysis()
        {
            string guid = "86cbfe43-175a-4a5a-ba89-073f3004e7df";//20 class mturk all pascal
            //string guid = "61bf62b5-d796-4b3c-9de9-9c67f04cf26f";// changed taskpage

            string configString = "Min_" + 10 + "_Max_" + 10 + "_Majority_" + 0.5 + "_Ratio_" + 0.5;

            SatyamAggregatedResultsTableAccess       aggDB      = new SatyamAggregatedResultsTableAccess();
            List <SatyamAggregatedResultsTableEntry> aggEntries = aggDB.getEntriesByGUID(guid);

            aggDB.close();

            SatyamResultsTableAccess       resDB   = new SatyamResultsTableAccess();
            List <SatyamResultsTableEntry> entries = resDB.getEntriesByGUID(guid);

            resDB.close();

            List <SatyamResultsTableEntry> filteredResults = new List <SatyamResultsTableEntry>();

            //foreach (SatyamResultsTableEntry entry in entries)
            //{
            //    SatyamResult res = JSonUtils.ConvertJSonToObject<SatyamResult>(entry.ResultString);


            //    string workerID = res.amazonInfo.WorkerID;
            //    string assignmentID = res.amazonInfo.AssignmentID;
            //    if (assignmentID == "" || assignmentID == "ASSIGNMENT_ID_NOT_AVAILABLE") continue;
            //    if (workerID == "") continue;

            //    /// im seg only
            //    ImageSegmentationResult res_imseg = JSonUtils.ConvertJSonToObject<ImageSegmentationResult>(res.TaskResult);
            //    if (res_imseg == null) continue;

            //    filteredResults.Add(entry);
            //}

            filteredResults = entries;

            SortedDictionary <DateTime, List <SatyamResultsTableEntry> > entriesBySubmitTime =
                SatyamResultValidationToolKit.SortResultsBySubmitTime_OneResultPerTurkerPerTask(filteredResults);

            SatyamResultsAnalysis.AggregationAnalysis(aggEntries, entriesBySubmitTime, null, null, guid, configString,
                                                      usePreApprovalResult: false,
                                                      approvalRatioThreshold: 0.5);
        }
Exemplo n.º 2
0
        private void populateRows()
        {
            SatyamJobSubmissionsTableAccess             dbaccess = new SatyamJobSubmissionsTableAccess();
            List <SatyamJobSubmissionsTableAccessEntry> entries  = dbaccess.getAllEntriesByUserID(UserId);

            dbaccess.close();
            List <TableRow> rows    = new List <TableRow>();
            int             maxShow = 10;

            for (int i = entries.Count - 1; i >= Math.Max(0, entries.Count - 1 - maxShow); i--)
            {
                TableRow row = new TableRow();
                row.JobGUID         = entries[i].JobGUID;
                row.JobTemplateType = entries[i].JobTemplateType;
                row.JobSubmitTime   = entries[i].JobSubmitTime.ToLocalTime().ToString();
                row.JobStatus       = entries[i].JobStatus;
                SatyamTaskTableAccess taskDB = new SatyamTaskTableAccess();
                int remainingTasks           = taskDB.getTasksCountByGUID(row.JobGUID);
                taskDB.close();

                row.TaskPending = remainingTasks.ToString();
                if (entries[i].JobProgress == "")
                {
                    row.JobProgress = "1";
                }
                else
                {
                    row.JobProgress = (1 - (double)remainingTasks / Convert.ToDouble(entries[i].JobProgress)).ToString("0.000");
                }


                int totalResults, totalAggregated;
                row.ApprovalRate    = SatyamResultsAnalysis.getPaymentRateByGUID(row.JobGUID, out totalResults, out totalAggregated).ToString("0.000");
                row.TotalResults    = totalResults.ToString();
                row.TotalAggregated = totalAggregated.ToString();

                rows.Add(row);
            }

            rpt.DataSource = rows;
            rpt.DataBind();
        }
        public static void AggregateWithParameterAndValidateSatyamVideoClassificationResultByGUID(string guid,
                                                                                                  int MinResults                    = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MIN_RESULTS_TO_AGGREGATE,
                                                                                                  int MaxResults                    = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MAX_RESULTS_TO_AGGREGATE,
                                                                                                  double MajorityThreshold          = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MAJORITY_THRESHOLD,
                                                                                                  bool prepareDataForTraining       = false, string outputDirectory = null,
                                                                                                  string confusingVideoListFilePath = null)
        {
            string configString = "Min_" + MinResults + "_Max_" + MaxResults + "_Thresh_" + MajorityThreshold;

            Console.WriteLine("Aggregating for param set " + configString);
            int noTerminatedTasks = 0;
            SatyamResultsTableAccess       resultsDB = new SatyamResultsTableAccess();
            List <SatyamResultsTableEntry> entries   = resultsDB.getEntriesByGUIDOrderByID(guid);

            resultsDB.close();


            SortedDictionary <DateTime, List <SatyamResultsTableEntry> > entriesBySubmitTime = SatyamResultValidationToolKit.SortResultsBySubmitTime(entries);

            int noTotalConverged = 0;
            int noCorrect        = 0;

            Dictionary <int, List <SatyamResultsTableEntry> > ResultsPerTask = new Dictionary <int, List <SatyamResultsTableEntry> >();
            List <int> aggregatedTasks = new List <int>();

            SortedDictionary <string, Dictionary <string, int> > confusionMatrix_res_groundtruth = new SortedDictionary <string, Dictionary <string, int> >();
            SortedDictionary <string, Dictionary <string, int> > confusionMatrix_groundtruth_res = new SortedDictionary <string, Dictionary <string, int> >();

            List <SatyamAggregatedResultsTableEntry> aggEntries = new List <SatyamAggregatedResultsTableEntry>();

            Dictionary <int, int> noResultsNeededForAggregation     = SatyamResultsAnalysis.getNoResultsNeededForAggregationFromLog(configString, guid);
            Dictionary <int, int> noResultsNeededForAggregation_new = new Dictionary <int, int>();

            foreach (DateTime t in entriesBySubmitTime.Keys)
            {
                //Console.WriteLine("Processing Results of time: {0}", t);
                foreach (SatyamResultsTableEntry entry in entriesBySubmitTime[t])
                {
                    SatyamResult satyamResult             = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
                    SatyamTask   task                     = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
                    SatyamJob    job                      = task.jobEntry;
                    string       fileName                 = URIUtilities.filenameFromURI(task.SatyamURI);
                    string       VideoCategoryGroundTruth = getVideoCategoryFromFileName(fileName);

                    int taskEntryID = entry.SatyamTaskTableEntryID;
                    if (aggregatedTasks.Contains(taskEntryID))
                    {
                        continue;
                    }

                    if (!ResultsPerTask.ContainsKey(taskEntryID))
                    {
                        ResultsPerTask.Add(taskEntryID, new List <SatyamResultsTableEntry>());
                    }

                    ResultsPerTask[taskEntryID].Add(entry);

                    // check log if enough results are collected
                    if (noResultsNeededForAggregation != null && noResultsNeededForAggregation.ContainsKey(taskEntryID) &&
                        ResultsPerTask[taskEntryID].Count < noResultsNeededForAggregation[taskEntryID])
                    {
                        continue;
                    }

                    //SingleObjectLabelingAggregatedResult aggResult = SingleObjectLabelingAggregator.getAggregatedResult(ResultsPerTask[taskEntryID], MinResults, MaxResult, MajorityThreshold);
                    string aggResultString = SingleObjectLabelingAggregator.GetAggregatedResultString(ResultsPerTask[taskEntryID], MinResults, MaxResults, MajorityThreshold);

                    if (aggResultString == null)
                    {
                        continue;
                    }

                    SatyamAggregatedResultsTableEntry aggEntry = new SatyamAggregatedResultsTableEntry();
                    aggEntry.JobGUID                = ResultsPerTask[taskEntryID][0].JobGUID;
                    aggEntry.JobTemplateType        = ResultsPerTask[taskEntryID][0].JobTemplateType;
                    aggEntry.SatyamTaskTableEntryID = taskEntryID;
                    aggEntry.UserID       = ResultsPerTask[taskEntryID][0].UserID;
                    aggEntry.ResultString = aggResultString;

                    /// aggregation happen
                    // record logs
                    if (noResultsNeededForAggregation == null || !noResultsNeededForAggregation.ContainsKey(taskEntryID))
                    {
                        noResultsNeededForAggregation_new.Add(taskEntryID, ResultsPerTask[taskEntryID].Count);
                    }

                    aggEntries.Add(aggEntry);
                    noTotalConverged++;
                    if (ResultsPerTask[taskEntryID].Count >= MaxResults)
                    {
                        noTerminatedTasks++;
                    }
                    aggregatedTasks.Add(taskEntryID);
                }
            }

            SatyamResultsAnalysis.RecordAggregationLog(noResultsNeededForAggregation_new, configString, guid);


            string outputmatFile = DirectoryConstants.defaultTempDirectory + guid + "\\" + configString + "_mat.txt";

            EvaluateAndPrintConfusionMatrixOfAggregatedResultEntries(aggEntries, outputmatFile, out noCorrect, prepareDataForTraining, outputDirectory);


            Console.WriteLine("noTotalConverged {0}", noTotalConverged);
            Console.WriteLine("noTerminatedTasks {0}", noTerminatedTasks);
            Console.WriteLine("Result: {0}/{1}, precision: {2}", noCorrect, noTotalConverged, (double)noCorrect / noTotalConverged);
            /// local file
            string outputString = String.Format("{0} {1} {2} {3} {4} {5}\n", configString, noCorrect, noTotalConverged, (double)noCorrect / noTotalConverged, noTerminatedTasks, ResultsPerTask.Count - noTotalConverged);
            string outputfile   = DirectoryConstants.defaultTempDirectory + guid + "\\resultSummary.txt";

            File.AppendAllText(outputfile, outputString);



            //for (double prob = 0; prob < 1;prob +=0.2)
            //{
            //    SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entries, guid, configString, anotherChanceProbablity: prob);
            //}
            //for (double ratio = 0; ratio < 1; ratio += 0.2)
            //{
            //    SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guid, configString, approvalRatioThreshold: ratio);
            //}
            SatyamResultsAnalysis.AggregationAnalysis(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guid, configString);
        }
Exemplo n.º 4
0
        public static void TestSatyamResultAnalysis()
        {
            List <List <string> > jobs = new List <List <string> >()
            {
                //// Video Classification
                //new List<string>(){ "ca2a97e3-3c32-48cc-b7d9-fdd945b50f23" }, //10 class human gesture in JHMDB

                //// Image Classification
                //new List<string>(){ "9c6283dd-f218-4588-a0d9-8c0afe841503" }, // all img 10class
                //new List<string>(){ "03977c5f-0e87-49ab-9c49-f20bfb47726a" }, // 2img 10 class

                //// Image Counting
                //new List<string>(){
                //    "94979929-4315-4c81-9782-55c8955222a8",// CARPK NTU 20161029
                //    "58a85bc8-4fb9-4672-8719-38cf62352fbc", // KITTI all
                //},

                //// Image Detection
                new List <string>()
                {
                    //"3609af9c-e734-41dc-997c-94c060a1f63d",
                    //"5f14ecaa-1c4c-41d6-ad05-6ee0929e494e",
                    //"8fbc6ca1-e6c0-4720-93a3-1d49aa873a65",
                    //"d6f1c3ee-cd59-4197-b40a-0f45512596eb",  // KITTI Detection all

                    //"a7e3fe3a-5097-4149-99bc-bae06da18aba", // rivision tasks satyam
                    //"aefd12fc-42ed-4ec5-8c9a-6c41703a843f", // revision task kitti

                    //"2c239102-e38f-4f0e-b639-6fae05250475", // without revision task kitti
                    //"2b1a86ee-7d39-49db-90bc-fb8d2ff63019", // revision kitti 10 internal test with recordings

                    //"a328f600-3b1f-4845-87a0-402648ad51b1", // revision Seattle 10, TF Serving backend
                    //"5c403d7f-6c48-4638-b545-472b9f190606", // no revision Seattle 10

                    //"40eee69b-c260-4760-940f-92f30cdfbb7f", // RevisionTF Seattle9000 on TestStewart 10
                    //"a55c9c21-16c0-432a-930e-de428d8213a5", // RevisionTF Fastrcnn_KITTI on TestStewart 10
                    //"9e3096f6-e898-45eb-8b15-e2854485a59c", // RevisionTF Fastrcnn_KITTI on TestStewart 30, but aggregation is not functioning for most of the time
                    "3e835157-d058-46ee-a13b-9c1566cba505", //RevisionTF Seattle9000 on TestStewart 30
                },

                ////// Video Tracking
                //new List<string>(){
                //    "1e43a983-548d-4a2e-8161-5537eb985902",
                //    "77c4f6af-f669-42fb-925a-4ff670cf9ae2"}, // KITTI Tracking all

                ////// Caprk Counting Adaptive Pricing Test
                //new List<string>(){
                //    //"f7c498d8-c06e-4b46-9169-71048c0e4eac",// carpk 10
                //    "e5fdff10-f018-4779-9d18-70c25fa3259f",}, // carpk all
                //new List<string>(){
                //    "a388df30-8742-4b6e-911d-32093502c2e8"}, // KITTI all

                //////// Image Segmentation
                //new List<string>()
                //{
                //    "86cbfe43-175a-4a5a-ba89-073f3004e7df", // pascal VOC
                //},

                ////// Cam Pose Estimation
                //new List<string>()
                //{
                //    "dff5415b-b278-43db-8e32-4f654ca4b3a1", //3000 results
                //}
            };


            foreach (List <string> guids in jobs)
            {
                SatyamResultsAnalysis.AnalyzeFromResultsTable(guids);
            }
        }
Exemplo n.º 5
0
        public static void AdaptTaskParameters()
        {
            double TargetPricePerHour = -1;
            double Budget             = -1;
            //double TargetPricePerHour = 13.56;
            //double MinPricePerTask = 0.01;
            //double MaxPricePerTask = 0.5;
            double MinPricePerTaskDifferenceToAdjust = 0.01;//USD
            int    MinNoResultsToAdjust = 10;

            Dictionary <string, List <SatyamTaskTableEntry> > MTurkTasksByGUID = OrganizeMTurkTasksByGUID();

            if (MTurkTasksByGUID == null)
            {
                return;
            }

            // get statistics
            foreach (string guid in MTurkTasksByGUID.Keys)
            {
                // testing for counting only
                List <string> ACGuids = new List <string>()
                {
                    "e5fdff10-f018-4779-9d18-70c25fa3259f", //carpk 164
                    "a388df30-8742-4b6e-911d-32093502c2e8", //kitti 100
                };
                if (!ACGuids.Contains(guid))
                {
                    continue;
                }

                SatyamTaskTableEntry task0 = MTurkTasksByGUID[guid][0];
                double CurrentPricePerTask = task0.PricePerTask;

                SatyamTask t = JSonUtils.ConvertJSonToObject <SatyamTask>(task0.TaskParametersString);
                TargetPricePerHour = t.jobEntry.TargetPricePerTask;
                //TargetPricePerHour = 13.56; //temporary
                Budget = t.jobEntry.TotalBudget;

                if (TargetPricePerHour <= 0)
                {
                    continue;                         //invalid
                }
                SatyamResultPerJobDataAnalysis ana = SatyamResultsAnalysis.AnalyzeFromResultsTable(new List <string>()
                {
                    guid
                });

                if (ana == null)
                {
                    continue;
                }

                int noAcceptedResults = ana.totalAcceptedResults;
                if (noAcceptedResults < MinNoResultsToAdjust)
                {
                    continue;
                }


                double medianTimePerTaskInSeconds = ana.medianAcceptedTimeTakenPerResultInSeconds;
                double TargetPricePerTask         = TargetPricePerHour / 3600 * medianTimePerTaskInSeconds;
                //TargetPricePerTask = Math.Max(TargetPricePerTask, MinPricePerTask);
                //TargetPricePerTask = Math.Min(TargetPricePerTask, MaxPricePerTask);
                TargetPricePerTask = Math.Round(TargetPricePerTask, 2);// round to cents.

                // is the price within the budget?
                double MinBudgetNeeded = MTurkTasksByGUID[guid].Count * TargetPricePerTask * 3;// at least 3 results per task.
                if (Budget < MinBudgetNeeded)
                {
                    Console.WriteLine("Budget too small to adapt price");
                    continue;
                }


                if (Math.Abs(CurrentPricePerTask - TargetPricePerTask) > MinPricePerTaskDifferenceToAdjust)
                {
                    // need to adjust
                    Console.WriteLine("Adapting the price to {0}", TargetPricePerTask);
                    ExpireHitByGUID(guid);
                    AdjustTasksByGUID(guid, TargetPricePerTask);
                }
            }
        }
Exemplo n.º 6
0
        public static void AggregateWithParameter(string guid,
                                                  int MinResults = TaskConstants.TRACKLET_LABELING_MTURK_MIN_RESULTS_TO_AGGREGATE,
                                                  int MaxResults = TaskConstants.TRACKLET_LABELING_MTURK_MAX_RESULTS_TO_AGGREGATE,
                                                  double boxToleranceThreshold = TaskConstants.TRACKLET_LABELING_BOX_DEVIATION_THRESHOLD,
                                                  double ObjectCoverageApprovalThresholdPerVideo = TaskConstants.TRACKLET_LABELING_APPROVALRATIO_PER_VIDEO,
                                                  double BoxCoverageApprovalThresholdPerTrack    = TaskConstants.TRACKLET_LABELING_APPROVALRATIO_PER_TRACK,
                                                  int consensusNumber = TaskConstants.TRACKLET_LABELING_MIN_RESULTS_FOR_CONSENSUS,
                                                  double minTubeletIoUSimilarityThreshold = TaskConstants.TRACKLET_LABELING_MIN_TUBELET_SIMILARITY_THRESHOLD,
                                                  double attributeMajority      = TaskConstants.TRACKLET_LABELING_MTURK_ATTRIBUTE_MAJORITY_THRESHOLD,
                                                  bool allFalseAttributeInvalid = false
                                                  )
        {
            string configString = "Min_" + MinResults + "_Max_" + MaxResults + "_IoU_" + minTubeletIoUSimilarityThreshold + "_Ratio_" + ObjectCoverageApprovalThresholdPerVideo;

            Console.WriteLine("Aggregating for " + guid + " with param set " + configString);
            SatyamResultsTableAccess       resultsDB = new SatyamResultsTableAccess();
            List <SatyamResultsTableEntry> entries   = resultsDB.getEntriesByGUID(guid);

            resultsDB.close();

            SortedDictionary <DateTime, List <SatyamResultsTableEntry> > entriesBySubmitTime = SatyamResultValidationToolKit.SortResultsBySubmitTime_OneResultPerTurkerPerTask(entries);

            Dictionary <int, List <MultiObjectTrackingResult> > ResultsPerTask = new Dictionary <int, List <MultiObjectTrackingResult> >();
            List <int> aggregatedTasks = new List <int>();

            int noTotalConverged = 0;
            //int noCorrect = 0;
            int noTerminatedTasks = 0;

            List <SatyamAggregatedResultsTableEntry> aggEntries = new List <SatyamAggregatedResultsTableEntry>();

            Dictionary <int, int> noResultsNeededForAggregation     = SatyamResultsAnalysis.getNoResultsNeededForAggregationFromLog(configString, guid);
            Dictionary <int, int> noResultsNeededForAggregation_new = new Dictionary <int, int>();
            // play back by time
            Dictionary <int, List <string> > WorkersPerTask = new Dictionary <int, List <string> >();

            foreach (DateTime t in entriesBySubmitTime.Keys)
            {
                //Console.WriteLine("Processing Results of time: {0}", t);
                List <SatyamResultsTableEntry> ResultEntries = entriesBySubmitTime[t];
                foreach (SatyamResultsTableEntry entry in ResultEntries)
                {
                    SatyamResult satyamResult           = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
                    SatyamTask   task                   = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
                    MultiObjectTrackingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(task.jobEntry.JobParameters);
                    string fileName    = URIUtilities.filenameFromURINoExtension(task.SatyamURI);
                    int    taskEntryID = entry.SatyamTaskTableEntryID;
                    if (aggregatedTasks.Contains(taskEntryID))
                    {
                        continue;
                    }
                    if (!ResultsPerTask.ContainsKey(taskEntryID))
                    {
                        ResultsPerTask.Add(taskEntryID, new List <MultiObjectTrackingResult>());
                        WorkersPerTask.Add(taskEntryID, new List <string>());
                    }

                    // remove duplicate workers result
                    string workerID = satyamResult.amazonInfo.WorkerID;
                    if (WorkersPerTask[taskEntryID].Contains(workerID))
                    {
                        continue;
                    }
                    //enclose only non-duplicate results, one per each worker.
                    WorkersPerTask[taskEntryID].Add(workerID);



                    string   videoName        = URIUtilities.localDirectoryNameFromURI(task.SatyamURI);
                    string[] fields           = videoName.Split('_');
                    int      startingFrame    = Convert.ToInt32(fields[fields.Length - 1]);
                    int      maxChunkEndFrame = startingFrame + job.ChunkDuration * job.FrameRate;
                    int      noFrameOverlap   = (int)(job.ChunkOverlap * job.FrameRate);
                    if (startingFrame != 0)
                    {
                        startingFrame -= noFrameOverlap;
                    }



                    string blobDir = URIUtilities.localDirectoryFullPathFromURI(task.SatyamURI);
                    VATIC_DVA_CrowdsourcedResult taskr = TrackletLabelingAggregator.createVATIC_DVA_CrowdsourcedResultUsingSatyamBlobImageCount(satyamResult.TaskResult, blobDir, entry.SatyamTaskTableEntryID.ToString(), job.FrameRate);
                    //MultiObjectTrackingResult result_tmp = vatic_tmp.getCompressedTracksInTimeSegment();

                    //VATIC_DVA_CrowdsourcedResult taskr = new VATIC_DVA_CrowdsourcedResult(satyamResult.TaskResult, videoName, entry.ID.ToString(), ImageURLs.Count, job.FrameRate);
                    MultiObjectTrackingResult res = taskr.getCompressedTracksInTimeSegment();

                    if (allFalseAttributeInvalid)
                    {
                        if (TrackletLabelingAggregator.AllAttributeAllFalse(res))
                        {
                            continue;
                        }
                    }

                    ResultsPerTask[taskEntryID].Add(res);

                    // check log if enough results are collected

                    if (noResultsNeededForAggregation != null && noResultsNeededForAggregation.ContainsKey(taskEntryID) &&
                        ResultsPerTask[taskEntryID].Count < noResultsNeededForAggregation[taskEntryID])
                    {
                        continue;
                    }



                    // hack for masters
                    int tempMin = MinResults;
                    if (TaskConstants.masterGUIDs.Contains(entry.JobGUID))
                    {
                        tempMin = 1;
                    }

                    TrackletLabelingAggregatedResult aggResult = TrackletLabelingAggregator.getAggregatedResult(ResultsPerTask[taskEntryID],
                                                                                                                guid, task.SatyamURI,
                                                                                                                job.FrameRate, job.BoundaryLines,
                                                                                                                MinResults, MaxResults, boxToleranceThreshold,
                                                                                                                ObjectCoverageApprovalThresholdPerVideo,
                                                                                                                BoxCoverageApprovalThresholdPerTrack,
                                                                                                                consensusNumber, minTubeletIoUSimilarityThreshold,
                                                                                                                attributeMajority);

                    if (aggResult == null)
                    {
                        continue;
                    }

                    //////////////// aggregation happen
                    // record logs
                    if (noResultsNeededForAggregation == null || !noResultsNeededForAggregation.ContainsKey(taskEntryID))
                    {
                        noResultsNeededForAggregation_new.Add(taskEntryID, ResultsPerTask[taskEntryID].Count);
                    }



                    aggregatedTasks.Add(taskEntryID);
                    noTotalConverged++;
                    if (ResultsPerTask[taskEntryID].Count >= MaxResults)
                    {
                        noTerminatedTasks++;
                    }
                    SatyamAggregatedResult SatyamAggResult = new SatyamAggregatedResult();
                    SatyamAggResult.SatyamTaskTableEntryID = taskEntryID;
                    SatyamAggResult.AggregatedResultString = JSonUtils.ConvertObjectToJSon <TrackletLabelingAggregatedResult>(aggResult);
                    SatyamAggResult.TaskParameters         = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString).TaskParametersString;

                    SatyamAggregatedResultsTableEntry aggEntry = new SatyamAggregatedResultsTableEntry();
                    aggEntry.SatyamTaskTableEntryID = taskEntryID;
                    aggEntry.JobGUID      = entry.JobGUID;
                    aggEntry.ResultString = JSonUtils.ConvertObjectToJSon <SatyamAggregatedResult>(SatyamAggResult);


                    aggEntries.Add(aggEntry);
                }
            }


            Console.WriteLine("Total_Aggregated_Tasks: {0}", noTotalConverged);
            Console.WriteLine("Total_Terminated_Tasks: {0}", noTerminatedTasks);

            SatyamResultsAnalysis.RecordAggregationLog(noResultsNeededForAggregation_new, configString, guid);

            TrackletLabelingAnalyzer.GroupEntriesByVideoNameAndStitchAndSaveAggregatedResultVideosLocally(aggEntries, DirectoryConstants.defaultTempDirectory + guid);
        }
Exemplo n.º 7
0
        public static void AggregateWithParameterAndValidateObjectCountingResultByGUID(string guid,
                                                                                       //bool saveImage = false,
                                                                                       bool prepareTrainingSet = false, string outputDirectory = null,
                                                                                       int MinHeight           = TaskConstants.OBJECT_COUNTING_VALIDATION_MIN_HEIGHT,
                                                                                       int MaxOcclusion        = TaskConstants.OBJECT_COUNTING_VALIDATION_MAX_OCCLUSION,
                                                                                       double Max_Truncation   = TaskConstants.OBJECT_COUNTING_VALIDATION_MIN_TRUNCATION,
                                                                                       int MinResults          = TaskConstants.OBJECT_COUNTING_MTURK_MIN_RESULTS_TO_AGGREGATE,
                                                                                       int MaxResults          = TaskConstants.OBJECT_COUNTING_MTURK_MAX_RESULTS_TO_AGGREGATE,
                                                                                       double MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND = TaskConstants.OBJECT_COUNTING_MTURK_MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND,
                                                                                       double MAX_DEVIATION_FRACTION = TaskConstants.OBJECT_COUNTING_MTURK_MAX_DEVIATION_FRACTION,
                                                                                       double SUPER_MAJORITY_VALUE   = TaskConstants.OBJECT_COUNTING_MTURK_SUPER_MAJORITY_VALUE
                                                                                       )
        {
            if (!Directory.Exists(DirectoryConstants.defaultTempDirectory + guid))
            {
                Directory.CreateDirectory(DirectoryConstants.defaultTempDirectory + guid);
            }

            string configString = "Min_" + MinResults + "_Max_" + MaxResults + "_Dev_" + MAX_DEVIATION_FRACTION + "_Majority_" + SUPER_MAJORITY_VALUE;

            Console.WriteLine("Aggregating for param set " + configString);
            int noTerminatedTasks = 0;
            SatyamResultsTableAccess       resultsDB = new SatyamResultsTableAccess();
            List <SatyamResultsTableEntry> entries   = resultsDB.getEntriesByGUIDOrderByID(guid);

            resultsDB.close();

            SortedDictionary <DateTime, List <SatyamResultsTableEntry> > entriesBySubmitTime = SatyamResultValidationToolKit.SortResultsBySubmitTime(entries);

            int noTotalConverged = 0;
            //int noCorrect = 0;

            Dictionary <int, List <SatyamResultsTableEntry> > ResultsPerTask = new Dictionary <int, List <SatyamResultsTableEntry> >();
            List <int> aggregatedTasks = new List <int>();

            List <SatyamAggregatedResultsTableEntry> aggEntries     = new List <SatyamAggregatedResultsTableEntry>();
            Dictionary <int, int> noResultsNeededForAggregation     = SatyamResultsAnalysis.getNoResultsNeededForAggregationFromLog(configString, guid);
            Dictionary <int, int> noResultsNeededForAggregation_new = new Dictionary <int, int>();

            // play back by time
            foreach (DateTime t in entriesBySubmitTime.Keys)
            {
                //Console.WriteLine("Processing Results of time: {0}", t);
                foreach (SatyamResultsTableEntry entry in entriesBySubmitTime[t])
                {
                    SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
                    SatyamTask   task         = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
                    SatyamJob    job          = task.jobEntry;
                    string       fileName     = URIUtilities.filenameFromURI(task.SatyamURI);


                    int taskEntryID = entry.SatyamTaskTableEntryID;
                    if (aggregatedTasks.Contains(taskEntryID))
                    {
                        continue;
                    }

                    if (!ResultsPerTask.ContainsKey(taskEntryID))
                    {
                        ResultsPerTask.Add(taskEntryID, new List <SatyamResultsTableEntry>());
                    }
                    ResultsPerTask[taskEntryID].Add(entry);

                    // check log if enough results are collected
                    if (noResultsNeededForAggregation != null && noResultsNeededForAggregation.ContainsKey(taskEntryID) &&
                        ResultsPerTask[taskEntryID].Count < noResultsNeededForAggregation[taskEntryID])
                    {
                        continue;
                    }

                    string aggResultString = ObjectCountingAggregator.GetAggregatedResultString(ResultsPerTask[taskEntryID], MinResults, MaxResults, MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND, MAX_DEVIATION_FRACTION, SUPER_MAJORITY_VALUE);

                    if (aggResultString == null)
                    {
                        continue;
                    }

                    SatyamAggregatedResultsTableEntry aggEntry = new SatyamAggregatedResultsTableEntry();
                    aggEntry.JobGUID                = ResultsPerTask[taskEntryID][0].JobGUID;
                    aggEntry.JobTemplateType        = ResultsPerTask[taskEntryID][0].JobTemplateType;
                    aggEntry.SatyamTaskTableEntryID = taskEntryID;
                    aggEntry.UserID       = ResultsPerTask[taskEntryID][0].UserID;
                    aggEntry.ResultString = aggResultString;
                    /// aggregation happen
                    /// // record logs
                    // record logs
                    if (noResultsNeededForAggregation == null || !noResultsNeededForAggregation.ContainsKey(taskEntryID))
                    {
                        noResultsNeededForAggregation_new.Add(taskEntryID, ResultsPerTask[taskEntryID].Count);
                    }
                    aggEntries.Add(aggEntry);
                    noTotalConverged++;
                    if (ResultsPerTask[taskEntryID].Count >= MaxResults)
                    {
                        noTerminatedTasks++;
                    }
                    aggregatedTasks.Add(taskEntryID);
                }
            }

            SatyamResultsAnalysis.RecordAggregationLog(noResultsNeededForAggregation_new, configString, guid);

            double totalError, totalGroundtruth;

            //ValidateSatyamKITTIObjectCountingAggregationResult(aggEntries, out totalError, out totalGroundtruth,
            //    MinHeight, MaxOcclusion, Max_Truncation);
            ValidateSatyamCARPKObjectCountingAggregationResult(aggEntries, out totalError, out totalGroundtruth);

            Console.WriteLine("noTotalConverged {0}", noTotalConverged);
            Console.WriteLine("noTerminatedTasks {0}", noTerminatedTasks);
            /// local file
            string outputString = String.Format("{0} {1} {2} {3} {4} {5} {6}\n", configString, totalError, totalGroundtruth, totalError / totalGroundtruth, noTotalConverged, noTerminatedTasks, ResultsPerTask.Count - noTotalConverged);
            string outputfile   = DirectoryConstants.defaultTempDirectory + guid + "\\resultSummary.txt";

            File.AppendAllText(outputfile, outputString);

            string approvalString = configString + "_PayDev_" + TaskConstants.OBJECT_COUNTING_MTURK_MAX_ABSOLUTE_COUNT_DEVIATION_FOR_PAYMENT +
                                    "_PayFrac_" + TaskConstants.OBJECT_COUNTING_MTURK_MAX_DEVIATION_FRACTION_FOR_PAYMENT;

            //for (double prob = 0; prob < 1;prob +=0.2)
            //{
            //    SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entries, guid, approvalString, anotherChanceProbablity: prob);
            //}
            //for (double ratio = 0; ratio < 1; ratio += 0.2)
            //{
            //    SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guid, configString, approvalRatioThreshold: ratio);
            //}
            SatyamResultsAnalysis.AggregationAnalysis(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guid, configString);
        }
Exemplo n.º 8
0
        public static void AggregateWithParameterAndValidateSatyamImageNetClassificationResultByGUID(string guid,
                                                                                                     int MinResults                    = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MIN_RESULTS_TO_AGGREGATE,
                                                                                                     int MaxResults                    = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MAX_RESULTS_TO_AGGREGATE,
                                                                                                     double MajorityThreshold          = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MAJORITY_THRESHOLD,
                                                                                                     string confusingImageListFilePath = null)
        {
            string configString = "Min_" + MinResults + "_Max_" + MaxResults + "_Thresh_" + MajorityThreshold;

            Console.WriteLine("Aggregating for param set " + configString);
            int noTerminatedTasks = 0;

            SatyamResultsTableAccess       resultsDB = new SatyamResultsTableAccess();
            List <SatyamResultsTableEntry> entries   = resultsDB.getEntriesByGUIDOrderByID(guid);

            resultsDB.close();

            //SortedDictionary<DateTime, List<SatyamResultsTableEntry>> entriesBySubmitTime =
            //    SatyamResultValidation.SortResultsBySubmitTime_OneResultPerTurkerPerTask(entries);
            SortedDictionary <DateTime, List <SatyamResultsTableEntry> > entriesBySubmitTime =
                SatyamResultValidationToolKit.SortResultsBySubmitTime(entries);

            Dictionary <string, ConfusingReason> imageBlackListReason = new Dictionary <string, ConfusingReason>();

            if (confusingImageListFilePath != null)
            {
                imageBlackListReason = getConfusingImageList(confusingImageListFilePath);
            }

            //Dictionary<int, List<SingleObjectLabelingResult>> ResultsPerTask = new Dictionary<int, List<SingleObjectLabelingResult>>();
            Dictionary <int, List <SatyamResultsTableEntry> > ResultsPerTask = new Dictionary <int, List <SatyamResultsTableEntry> >();

            List <int> aggregatedTasks = new List <int>();

            int noTotalConverged = 0;
            int noCorrect        = 0;

            SortedDictionary <string, Dictionary <string, int> > confusionMatrix_res_groundtruth = new SortedDictionary <string, Dictionary <string, int> >();
            SortedDictionary <string, Dictionary <string, int> > confusionMatrix_groundtruth_res = new SortedDictionary <string, Dictionary <string, int> >();

            StringBuilder s = new StringBuilder();

            s.Append("<!DOCTYPE html>\n");
            s.Append("<html>\n");
            s.Append("<body>\n");
            String title = String.Format("<h1>Job GUID {0} Incorrect Result Summary</h1>\n", guid);

            s.Append(title);

            Dictionary <int, int> noResultsNeededForAggregation     = SatyamResultsAnalysis.getNoResultsNeededForAggregationFromLog(configString, guid);
            Dictionary <int, int> noResultsNeededForAggregation_new = new Dictionary <int, int>();

            List <SatyamAggregatedResultsTableEntry> aggEntries = new List <SatyamAggregatedResultsTableEntry>();

            foreach (DateTime t in entriesBySubmitTime.Keys)
            {
                foreach (SatyamResultsTableEntry entry in entriesBySubmitTime[t])
                {
                    //SatyamResultsTableEntry entry = entries[i];
                    SatyamResult satyamResult      = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
                    SatyamTask   task              = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
                    SatyamJob    job               = task.jobEntry;
                    string       fileName          = URIUtilities.filenameFromURI(task.SatyamURI);
                    string       imageCategoryName = fileName.Split('_')[0];

                    if (IsBlackListed(task.SatyamURI, imageBlackListReason))
                    {
                        continue;
                    }
                    int taskEntryID = entry.SatyamTaskTableEntryID;
                    if (aggregatedTasks.Contains(taskEntryID))
                    {
                        continue;
                    }

                    if (!ResultsPerTask.ContainsKey(taskEntryID))
                    {
                        ResultsPerTask.Add(taskEntryID, new List <SatyamResultsTableEntry>());
                    }
                    //ResultEntriesPerTask[taskEntryID].Add(JSonUtils.ConvertJSonToObject<SingleObjectLabelingResult>(satyamResult.TaskResult));
                    ResultsPerTask[taskEntryID].Add(entry);

                    // check log if enough results are collected
                    if (noResultsNeededForAggregation != null && noResultsNeededForAggregation.ContainsKey(taskEntryID) &&
                        ResultsPerTask[taskEntryID].Count < noResultsNeededForAggregation[taskEntryID])
                    {
                        continue;
                    }


                    string aggResultString = SingleObjectLabelingAggregator.GetAggregatedResultString(ResultsPerTask[taskEntryID], MinResults, MaxResults, MajorityThreshold);
                    //SingleObjectLabelingAggregatedResult aggResult = SingleObjectLabelingAggregator.getAggregatedResult(ResultEntriesPerTask[taskEntryID], MinResults, MaxResults, MajorityThreshold);


                    if (aggResultString == null)
                    {
                        continue;
                    }



                    SatyamAggregatedResultsTableEntry aggEntry = new SatyamAggregatedResultsTableEntry();
                    aggEntry.JobGUID                = ResultsPerTask[taskEntryID][0].JobGUID;
                    aggEntry.JobTemplateType        = ResultsPerTask[taskEntryID][0].JobTemplateType;
                    aggEntry.SatyamTaskTableEntryID = ResultsPerTask[taskEntryID][0].SatyamTaskTableEntryID;
                    aggEntry.UserID       = ResultsPerTask[taskEntryID][0].UserID;
                    aggEntry.ResultString = aggResultString;

                    /// aggregation happen
                    // record logs
                    if (noResultsNeededForAggregation == null || !noResultsNeededForAggregation.ContainsKey(taskEntryID))
                    {
                        noResultsNeededForAggregation_new.Add(taskEntryID, ResultsPerTask[taskEntryID].Count);
                    }
                    ///
                    //if (aggResult.Category == "None of the Above")
                    //{
                    //    continue;
                    //}

                    aggEntries.Add(aggEntry);

                    SatyamSaveAggregatedDataSatyam data = new SatyamSaveAggregatedDataSatyam(aggEntry);
                    String resultString = data.AggregatedResultString;
                    SingleObjectLabelingAggregatedResult aggResult = JSonUtils.ConvertJSonToObject <SingleObjectLabelingAggregatedResult>(resultString);

                    if (!confusionMatrix_res_groundtruth.ContainsKey(aggResult.Category))
                    {
                        confusionMatrix_res_groundtruth.Add(aggResult.Category, new Dictionary <string, int>()
                        {
                            { GroundTruth[imageCategoryName], 0 }
                        });
                    }
                    else
                    {
                        if (!confusionMatrix_res_groundtruth[aggResult.Category].ContainsKey(GroundTruth[imageCategoryName]))
                        {
                            confusionMatrix_res_groundtruth[aggResult.Category].Add(GroundTruth[imageCategoryName], 0);
                        }
                    }

                    if (!confusionMatrix_groundtruth_res.ContainsKey(GroundTruth[imageCategoryName]))
                    {
                        confusionMatrix_groundtruth_res.Add(GroundTruth[imageCategoryName], new Dictionary <string, int>()
                        {
                            { aggResult.Category, 0 }
                        });
                    }
                    else
                    {
                        if (!confusionMatrix_groundtruth_res[GroundTruth[imageCategoryName]].ContainsKey(aggResult.Category))
                        {
                            confusionMatrix_groundtruth_res[GroundTruth[imageCategoryName]].Add(aggResult.Category, 0);
                        }
                    }
                    if (aggResult.Category.Equals(GroundTruth[imageCategoryName], StringComparison.InvariantCultureIgnoreCase))
                    {
                        noCorrect++;
                    }
                    else
                    {
                        //Console.WriteLine("{0}, Groundtruth: {1}, Aggregated: {2}, Votes: {3}",
                        //    fileName, GroundTruth[imageCategoryName], aggResult.Category,
                        //    JSonUtils.ConvertObjectToJSon(aggResult.metaData));

                        String record = String.Format("<p>{0}, Groundtruth: {1}, Aggregated: {2}, Votes: {3}</p>\n",
                                                      fileName, GroundTruth[imageCategoryName], aggResult.Category,
                                                      JSonUtils.ConvertObjectToJSon(aggResult.metaData));
                        String img = String.Format("<img src=\"{0}\" >\n", task.SatyamURI);
                        s.Append(record);
                        s.Append(img);
                    }
                    noTotalConverged++;
                    if (ResultsPerTask[taskEntryID].Count >= MaxResults)
                    {
                        noTerminatedTasks++;
                    }
                    confusionMatrix_res_groundtruth[aggResult.Category][GroundTruth[imageCategoryName]]++;
                    confusionMatrix_groundtruth_res[GroundTruth[imageCategoryName]][aggResult.Category]++;
                    aggregatedTasks.Add(taskEntryID);
                }
            }

            SatyamResultsAnalysis.RecordAggregationLog(noResultsNeededForAggregation_new, configString, guid);

            s.Append("</body>\n");
            s.Append("</html>\n");
            string dataToBeSaved = s.ToString();

            SatyamJobStorageAccountAccess storage = new SatyamJobStorageAccountAccess();
            string FileName = String.Format("AggregatedIncorrectResults-{0}_Min{1}Max{2}Thresh{3}.html", guid, MinResults, MaxResults, MajorityThreshold);

            storage.SaveATextFile("singleobjectlabeling", guid, FileName, dataToBeSaved);

            s.Clear();
            s.Append("<!DOCTYPE html>\n");
            s.Append("<html>\n");
            s.Append("<body>\n");
            s.Append(title);

            string resultSummary = String.Format("<p>Result: {0}/{1}, precision: {2}</p>\n", noCorrect, noTotalConverged, (double)noCorrect / noTotalConverged);

            resultSummary += String.Format("<p>Terminated: {0}, Not Enough Results: {1}</p>\n", noTerminatedTasks, ResultsPerTask.Count - noTotalConverged);
            Console.WriteLine(resultSummary);
            s.Append(resultSummary);

            // write the confusion matrix
            s.Append("<p>");
            String row = "\t\t";

            foreach (string resultCategory in confusionMatrix_res_groundtruth.Keys)
            {
                row += resultCategory + "\t";
            }
            row += "<br>\n";
            s.Append(row);
            Console.WriteLine(row);

            string matString = "";

            foreach (string groundTruthCategory in confusionMatrix_groundtruth_res.Keys)
            {
                row = groundTruthCategory + "\t";
                foreach (string resultCategory in confusionMatrix_res_groundtruth.Keys)
                {
                    if (confusionMatrix_groundtruth_res[groundTruthCategory].ContainsKey(resultCategory))
                    {
                        row += confusionMatrix_groundtruth_res[groundTruthCategory][resultCategory].ToString();
                    }
                    else
                    {
                        row += "0";
                    }
                    row += "\t";
                }
                row += "<br>\n";
                s.Append(row);
                Console.WriteLine(row);
                matString += row;
            }

            s.Append("</p>\n");

            s.Append("</body>\n");
            s.Append("</html>\n");

            string summaryToBeSaved = s.ToString();

            FileName = String.Format("Aggregated_Summary-{0}_Min{1}Max{2}Thresh{3}.html", guid, MinResults, MaxResults, MajorityThreshold);
            storage.SaveATextFile("singleobjectlabeling", guid, FileName, summaryToBeSaved);

            /// local file
            string outputString = String.Format("{0} {1} {2} {3} {4} {5}\n", configString, noCorrect, noTotalConverged, (double)noCorrect / noTotalConverged, noTerminatedTasks, ResultsPerTask.Count - noTotalConverged);
            string outputfile   = DirectoryConstants.defaultTempDirectory + guid + "\\resultSummary.txt";

            File.AppendAllText(outputfile, outputString);
            string outputmatFile = DirectoryConstants.defaultTempDirectory + guid + "\\" + configString + "_mat.txt";

            File.WriteAllText(outputmatFile, matString);



            //for (double prob = 0; prob < 1;prob +=0.2)
            //{
            //    SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entries, guid, configString, anotherChanceProbablity: prob);
            //}
            //for (double ratio = 0; ratio < 1; ratio += 0.2)
            //{
            //    SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guid, configString, approvalRatioThreshold: ratio);
            //}
            SatyamResultsAnalysis.AggregationAnalysis(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guid, configString);
        }
        /// <summary>
        /// function generalizable.. TODO... to all tasks
        /// </summary>
        /// <param name="guids"></param>
        /// <param name="IoUTreshold"></param>
        /// <param name="saveImage"></param>
        /// <param name="outputDirectory"></param>
        /// <param name="MinResults"></param>
        /// <param name="MaxResults"></param>
        /// <param name="CategoryMajorityThreshold"></param>
        /// <param name="PolygonBoundaryMajorityThreshold"></param>
        /// <param name="ObjectsCoverageThreshold"></param>
        /// <param name="overwrite"></param>
        /// <param name="approvalAnalysis"></param>
        public static void AggregateWithParameterAndValidatePascalVOCImageSegmentationByGUID(List <string> guids,
                                                                                             double IoUTreshold,
                                                                                             bool saveImage = false, string outputDirectory = null,
                                                                                             int MinResults = TaskConstants.IMAGE_SEGMENTATION_MTURK_MIN_RESULTS_TO_AGGREGATE,
                                                                                             int MaxResults = TaskConstants.IMAGE_SEGMENTATION_MTURK_MAX_RESULTS_TO_AGGREGATE,
                                                                                             double CategoryMajorityThreshold        = TaskConstants.IMAGE_SEGMENTATION_MTURK_MAJORITY_CATEGORY_THRESHOLD,
                                                                                             double PolygonBoundaryMajorityThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_MAJORITY_POLYGON_BOUNDARY_THRESHOLD,
                                                                                             double ObjectsCoverageThreshold         = TaskConstants.IMAGE_SEGMENTATION_MTURK_OBJECT_COVERAGE_THRESHOLD_FOR_AGGREGATION_TERMINATION,
                                                                                             double minSimilarityThreshold           = TaskConstants.IMAGE_SEGMENTATION_MTURK_POLYGON_IOU_THRESHOLD,
                                                                                             int minResultsForConsensus = TaskConstants.IMAGE_SEGMENTATION_MTURK_MIN_RESULTS_FOR_CONSENSUS,
                                                                                             bool overwrite             = false,
                                                                                             bool approvalAnalysis      = false)
        {
            string configString = "Min_" + MinResults + "_Max_" + MaxResults + "_Majority_" + PolygonBoundaryMajorityThreshold + "_Ratio_" + ObjectsCoverageThreshold;

            Console.WriteLine("Aggregating for param set " + configString);
            if (!overwrite && File.Exists(DirectoryConstants.defaultTempDirectory + "\\ImageSegmentationResult\\" + configString + ".txt"))
            {
                return;
            }

            SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();

            Dictionary <int, List <string> > WorkersPerTask = new Dictionary <int, List <string> >();

            List <SatyamResultsTableEntry> entries = new List <SatyamResultsTableEntry>();

            foreach (string guid in guids)
            {
                entries.AddRange(resultsDB.getEntriesByGUIDOrderByID(guid));
            }
            resultsDB.close();

            SortedDictionary <DateTime, List <SatyamResultsTableEntry> > entriesBySubmitTime =
                SatyamResultValidationToolKit.SortResultsBySubmitTime_OneResultPerTurkerPerTask(entries);

            Dictionary <int, List <ImageSegmentationResult> > ResultsPerTask = new Dictionary <int, List <ImageSegmentationResult> >();
            List <int> aggregatedTasks = new List <int>();

            int noTotalConverged = 0;
            //int noCorrect = 0;
            int noTerminatedTasks = 0;

            List <SatyamAggregatedResultsTableEntry> aggEntries     = new List <SatyamAggregatedResultsTableEntry>();
            Dictionary <int, int> noResultsNeededForAggregation     = SatyamResultsAnalysis.getNoResultsNeededForAggregationFromLog(configString, guids[0]);
            Dictionary <int, int> noResultsNeededForAggregation_new = new Dictionary <int, int>();

            // play back by time
            foreach (DateTime t in entriesBySubmitTime.Keys)
            {
                //Console.WriteLine("Processing Results of time: {0}", t);
                List <SatyamResultsTableEntry> ResultEntries = entriesBySubmitTime[t];
                foreach (SatyamResultsTableEntry entry in ResultEntries)
                {
                    SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
                    SatyamTask   task         = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
                    SatyamJob    job          = task.jobEntry;
                    string       fileName     = URIUtilities.filenameFromURINoExtension(task.SatyamURI);
                    int          taskEntryID  = entry.SatyamTaskTableEntryID;
                    if (aggregatedTasks.Contains(taskEntryID))
                    {
                        continue;
                    }
                    if (!ResultsPerTask.ContainsKey(taskEntryID))
                    {
                        ResultsPerTask.Add(taskEntryID, new List <ImageSegmentationResult>());
                    }

                    ResultsPerTask[taskEntryID].Add(JSonUtils.ConvertJSonToObject <ImageSegmentationResult>(satyamResult.TaskResult));

                    // check log if enough results are collected
                    if (noResultsNeededForAggregation != null && noResultsNeededForAggregation.ContainsKey(taskEntryID) &&
                        ResultsPerTask[taskEntryID].Count < noResultsNeededForAggregation[taskEntryID])
                    {
                        continue;
                    }

                    ImageSegmentationAggregatedResult aggResult = ImageSegmentationAggregator.getAggregatedResult(
                        ResultsPerTask[taskEntryID], task.SatyamURI, entry.JobGUID,
                        MinResults, MaxResults, CategoryMajorityThreshold,
                        PolygonBoundaryMajorityThreshold, ObjectsCoverageThreshold,
                        minSimilarityThreshold, minResultsForConsensus);
                    if (aggResult == null)
                    {
                        continue;
                    }

                    // aggregation happen
                    // record logs
                    if (noResultsNeededForAggregation == null || !noResultsNeededForAggregation.ContainsKey(taskEntryID))
                    {
                        noResultsNeededForAggregation_new.Add(taskEntryID, ResultsPerTask[taskEntryID].Count);
                    }
                    aggregatedTasks.Add(taskEntryID);
                    noTotalConverged++;
                    if (ResultsPerTask[taskEntryID].Count >= MaxResults)
                    {
                        noTerminatedTasks++;
                    }
                    SatyamAggregatedResult SatyamAggResult = new SatyamAggregatedResult();
                    SatyamAggResult.SatyamTaskTableEntryID = taskEntryID;
                    SatyamAggResult.AggregatedResultString = JSonUtils.ConvertObjectToJSon <ImageSegmentationAggregatedResult>(aggResult);
                    SatyamAggResult.TaskParameters         = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString).TaskParametersString;

                    SatyamAggregatedResultsTableEntry aggEntry = new SatyamAggregatedResultsTableEntry();
                    aggEntry.SatyamTaskTableEntryID = taskEntryID;
                    aggEntry.JobGUID      = entry.JobGUID;
                    aggEntry.ResultString = JSonUtils.ConvertObjectToJSon <SatyamAggregatedResult>(SatyamAggResult);


                    aggEntries.Add(aggEntry);
                    List <SatyamAggregatedResultsTableEntry> tmpEntries = new List <SatyamAggregatedResultsTableEntry>();
                    tmpEntries.Add(aggEntry);

                    //ValidateSatyamKITTIDetectionAggregationResult(tmpEntries, saveImage, MinHeight, MaxOcclusion, Max_Truncation);
                }
            }
            Console.WriteLine("Total_Aggregated_Tasks: {0}", noTotalConverged);
            Console.WriteLine("Total_Terminated_Tasks: {0}", noTerminatedTasks);

            SatyamResultsAnalysis.RecordAggregationLog(noResultsNeededForAggregation_new, configString, guids[0]);

            string r = ValidatePascalVOCImageSegmentationResult_InstanceLevel(aggEntries, IoUTreshold);

            r = noTotalConverged + " " + noTerminatedTasks + " " + r;
            File.WriteAllText(DirectoryConstants.defaultTempDirectory + "\\ImageSegmentationResult\\" + configString + ".txt", r);


            if (approvalAnalysis)
            {
                string approvalString = configString + "_PayCover_" + TaskConstants.IMAGE_SEGMENTATION_MTURK_OBJECT_COVERAGE_THRESHOLD_FOR_PAYMENT +
                                        "_PayIoU_" + TaskConstants.IMAGE_SEGMENTATION_MTURK_POLYGON_IOU_THRESHOLD_FOR_PAYMENT;
                //for (double ratio = 0; ratio < 1; ratio += 0.2)
                //{
                //    SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guids[0], configString, approvalRatioThreshold: ratio);
                //}
                SatyamResultsAnalysis.AggregationAnalysis(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guids[0], configString);
            }
        }