public static bool IsAcceptable(SatyamAggregatedResultsTableEntry aggEntry, SatyamResultsTableEntry result)
        {
            switch (result.JobTemplateType)
            {
            case TaskConstants.Classification_Image:
            case TaskConstants.Classification_Image_MTurk:
            case TaskConstants.Classification_Video:
            case TaskConstants.Classification_Video_MTurk:
                return(SingleObjectLabelingAggregator.IsAcceptable(aggEntry, result));

            case TaskConstants.Counting_Image:
            case TaskConstants.Counting_Image_MTurk:
            case TaskConstants.Counting_Video:
            case TaskConstants.Counting_Video_MTurk:
                return(ObjectCountingAggregator.IsAcceptable(aggEntry, result));

            case TaskConstants.Detection_Image:
            case TaskConstants.Detection_Image_MTurk:
                return(MultiObjectLocalizationAndLabelingAggregator.IsAcceptable(aggEntry, result));

            case TaskConstants.Tracking:
            case TaskConstants.Tracking_MTurk:
                return(MultiObjectTrackingAggregator.IsAcceptable(aggEntry, result));

            case TaskConstants.Segmentation_Image:
            case TaskConstants.Segmentation_Image_MTurk:
                return(ImageSegmentationAggregator.IsAcceptable(aggEntry, result));

            case TaskConstants.OpenEndedQuestion_Image:
            case TaskConstants.OpenEndedQuestion_Image_MTurk:
                return(true);    //TBD

            case TaskConstants.TrackletLabeling:
            case TaskConstants.TrackletLabeling_MTurk:
                return(TrackletLabelingAggregator.IsAcceptable(aggEntry, result));
            }
            return(false);
        }
예제 #2
0
        public static void AggregateWithParameter(string guid,
                                                  int MinResults = TaskConstants.TRACKLET_LABELING_MTURK_MIN_RESULTS_TO_AGGREGATE,
                                                  int MaxResults = TaskConstants.TRACKLET_LABELING_MTURK_MAX_RESULTS_TO_AGGREGATE,
                                                  double boxToleranceThreshold = TaskConstants.TRACKLET_LABELING_BOX_DEVIATION_THRESHOLD,
                                                  double ObjectCoverageApprovalThresholdPerVideo = TaskConstants.TRACKLET_LABELING_APPROVALRATIO_PER_VIDEO,
                                                  double BoxCoverageApprovalThresholdPerTrack    = TaskConstants.TRACKLET_LABELING_APPROVALRATIO_PER_TRACK,
                                                  int consensusNumber = TaskConstants.TRACKLET_LABELING_MIN_RESULTS_FOR_CONSENSUS,
                                                  double minTubeletIoUSimilarityThreshold = TaskConstants.TRACKLET_LABELING_MIN_TUBELET_SIMILARITY_THRESHOLD,
                                                  double attributeMajority      = TaskConstants.TRACKLET_LABELING_MTURK_ATTRIBUTE_MAJORITY_THRESHOLD,
                                                  bool allFalseAttributeInvalid = false
                                                  )
        {
            string configString = "Min_" + MinResults + "_Max_" + MaxResults + "_IoU_" + minTubeletIoUSimilarityThreshold + "_Ratio_" + ObjectCoverageApprovalThresholdPerVideo;

            Console.WriteLine("Aggregating for " + guid + " with param set " + configString);
            SatyamResultsTableAccess       resultsDB = new SatyamResultsTableAccess();
            List <SatyamResultsTableEntry> entries   = resultsDB.getEntriesByGUID(guid);

            resultsDB.close();

            SortedDictionary <DateTime, List <SatyamResultsTableEntry> > entriesBySubmitTime = SatyamResultValidationToolKit.SortResultsBySubmitTime_OneResultPerTurkerPerTask(entries);

            Dictionary <int, List <MultiObjectTrackingResult> > ResultsPerTask = new Dictionary <int, List <MultiObjectTrackingResult> >();
            List <int> aggregatedTasks = new List <int>();

            int noTotalConverged = 0;
            //int noCorrect = 0;
            int noTerminatedTasks = 0;

            List <SatyamAggregatedResultsTableEntry> aggEntries = new List <SatyamAggregatedResultsTableEntry>();

            Dictionary <int, int> noResultsNeededForAggregation     = SatyamResultsAnalysis.getNoResultsNeededForAggregationFromLog(configString, guid);
            Dictionary <int, int> noResultsNeededForAggregation_new = new Dictionary <int, int>();
            // play back by time
            Dictionary <int, List <string> > WorkersPerTask = new Dictionary <int, List <string> >();

            foreach (DateTime t in entriesBySubmitTime.Keys)
            {
                //Console.WriteLine("Processing Results of time: {0}", t);
                List <SatyamResultsTableEntry> ResultEntries = entriesBySubmitTime[t];
                foreach (SatyamResultsTableEntry entry in ResultEntries)
                {
                    SatyamResult satyamResult           = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
                    SatyamTask   task                   = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
                    MultiObjectTrackingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(task.jobEntry.JobParameters);
                    string fileName    = URIUtilities.filenameFromURINoExtension(task.SatyamURI);
                    int    taskEntryID = entry.SatyamTaskTableEntryID;
                    if (aggregatedTasks.Contains(taskEntryID))
                    {
                        continue;
                    }
                    if (!ResultsPerTask.ContainsKey(taskEntryID))
                    {
                        ResultsPerTask.Add(taskEntryID, new List <MultiObjectTrackingResult>());
                        WorkersPerTask.Add(taskEntryID, new List <string>());
                    }

                    // remove duplicate workers result
                    string workerID = satyamResult.amazonInfo.WorkerID;
                    if (WorkersPerTask[taskEntryID].Contains(workerID))
                    {
                        continue;
                    }
                    //enclose only non-duplicate results, one per each worker.
                    WorkersPerTask[taskEntryID].Add(workerID);



                    string   videoName        = URIUtilities.localDirectoryNameFromURI(task.SatyamURI);
                    string[] fields           = videoName.Split('_');
                    int      startingFrame    = Convert.ToInt32(fields[fields.Length - 1]);
                    int      maxChunkEndFrame = startingFrame + job.ChunkDuration * job.FrameRate;
                    int      noFrameOverlap   = (int)(job.ChunkOverlap * job.FrameRate);
                    if (startingFrame != 0)
                    {
                        startingFrame -= noFrameOverlap;
                    }



                    string blobDir = URIUtilities.localDirectoryFullPathFromURI(task.SatyamURI);
                    VATIC_DVA_CrowdsourcedResult taskr = TrackletLabelingAggregator.createVATIC_DVA_CrowdsourcedResultUsingSatyamBlobImageCount(satyamResult.TaskResult, blobDir, entry.SatyamTaskTableEntryID.ToString(), job.FrameRate);
                    //MultiObjectTrackingResult result_tmp = vatic_tmp.getCompressedTracksInTimeSegment();

                    //VATIC_DVA_CrowdsourcedResult taskr = new VATIC_DVA_CrowdsourcedResult(satyamResult.TaskResult, videoName, entry.ID.ToString(), ImageURLs.Count, job.FrameRate);
                    MultiObjectTrackingResult res = taskr.getCompressedTracksInTimeSegment();

                    if (allFalseAttributeInvalid)
                    {
                        if (TrackletLabelingAggregator.AllAttributeAllFalse(res))
                        {
                            continue;
                        }
                    }

                    ResultsPerTask[taskEntryID].Add(res);

                    // check log if enough results are collected

                    if (noResultsNeededForAggregation != null && noResultsNeededForAggregation.ContainsKey(taskEntryID) &&
                        ResultsPerTask[taskEntryID].Count < noResultsNeededForAggregation[taskEntryID])
                    {
                        continue;
                    }



                    // hack for masters
                    int tempMin = MinResults;
                    if (TaskConstants.masterGUIDs.Contains(entry.JobGUID))
                    {
                        tempMin = 1;
                    }

                    TrackletLabelingAggregatedResult aggResult = TrackletLabelingAggregator.getAggregatedResult(ResultsPerTask[taskEntryID],
                                                                                                                guid, task.SatyamURI,
                                                                                                                job.FrameRate, job.BoundaryLines,
                                                                                                                MinResults, MaxResults, boxToleranceThreshold,
                                                                                                                ObjectCoverageApprovalThresholdPerVideo,
                                                                                                                BoxCoverageApprovalThresholdPerTrack,
                                                                                                                consensusNumber, minTubeletIoUSimilarityThreshold,
                                                                                                                attributeMajority);

                    if (aggResult == null)
                    {
                        continue;
                    }

                    //////////////// aggregation happen
                    // record logs
                    if (noResultsNeededForAggregation == null || !noResultsNeededForAggregation.ContainsKey(taskEntryID))
                    {
                        noResultsNeededForAggregation_new.Add(taskEntryID, ResultsPerTask[taskEntryID].Count);
                    }



                    aggregatedTasks.Add(taskEntryID);
                    noTotalConverged++;
                    if (ResultsPerTask[taskEntryID].Count >= MaxResults)
                    {
                        noTerminatedTasks++;
                    }
                    SatyamAggregatedResult SatyamAggResult = new SatyamAggregatedResult();
                    SatyamAggResult.SatyamTaskTableEntryID = taskEntryID;
                    SatyamAggResult.AggregatedResultString = JSonUtils.ConvertObjectToJSon <TrackletLabelingAggregatedResult>(aggResult);
                    SatyamAggResult.TaskParameters         = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString).TaskParametersString;

                    SatyamAggregatedResultsTableEntry aggEntry = new SatyamAggregatedResultsTableEntry();
                    aggEntry.SatyamTaskTableEntryID = taskEntryID;
                    aggEntry.JobGUID      = entry.JobGUID;
                    aggEntry.ResultString = JSonUtils.ConvertObjectToJSon <SatyamAggregatedResult>(SatyamAggResult);


                    aggEntries.Add(aggEntry);
                }
            }


            Console.WriteLine("Total_Aggregated_Tasks: {0}", noTotalConverged);
            Console.WriteLine("Total_Terminated_Tasks: {0}", noTerminatedTasks);

            SatyamResultsAnalysis.RecordAggregationLog(noResultsNeededForAggregation_new, configString, guid);

            TrackletLabelingAnalyzer.GroupEntriesByVideoNameAndStitchAndSaveAggregatedResultVideosLocally(aggEntries, DirectoryConstants.defaultTempDirectory + guid);
        }
        //public static void Aggregate()
        //{
        //    //first get all the results that are not aggregated
        //    SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
        //    List<SatyamResultsTableEntry> results = resultsDB.getEntriesByStatus();

        //    //first group all the results by the tasks since wach task may ahve multiple entries
        //    Dictionary<int, List<SatyamResultsTableEntry>> collectedResults = new Dictionary<int, List<SatyamResultsTableEntry>>();
        //    foreach (SatyamResultsTableEntry entry in results)
        //    {
        //        int taskID = entry.SatyamTaskTableEntryID;
        //        if (!collectedResults.ContainsKey(taskID))
        //        {
        //            collectedResults.Add(taskID, new List<SatyamResultsTableEntry>());
        //        }
        //        collectedResults[taskID].Add(entry);
        //    }

        //    //now aggreagate the results for eacht ask.
        //    List<int> taskIDList = collectedResults.Keys.ToList();
        //    List<SatyamAggregatedResultsTableEntry> aggEntries = new List<SatyamAggregatedResultsTableEntry>();
        //    foreach (int taskId in taskIDList)
        //    {
        //        SatyamAggregatedResultsTableEntry aggEntry = GetAggregatedResultString(taskId, collectedResults[taskId]);
        //        if (aggEntry != null)
        //        {
        //            aggEntries.Add(aggEntry);
        //        }
        //    }
        //    SatyamAggregatedResultsTableAccess aggDB = new SatyamAggregatedResultsTableAccess();
        //    aggDB.AddEntries(aggEntries);
        //    resultsDB.close();
        //    aggDB.close();
        //}

        public static SatyamAggregatedResultsTableEntry GetAggregatedResultString(int taskId, List <SatyamResultsTableEntry> resultEntries)
        {
            SatyamAggregatedResultsTableEntry aggEntry = null;
            string templateType    = resultEntries[0].JobTemplateType;
            string aggResultString = null;

            switch (templateType)
            {
            case TaskConstants.Classification_Image:
            case TaskConstants.Classification_Image_MTurk:
            case TaskConstants.Classification_Video:
            case TaskConstants.Classification_Video_MTurk:
                aggResultString = SingleObjectLabelingAggregator.GetAggregatedResultString(resultEntries);
                break;

            case TaskConstants.Counting_Image:
            case TaskConstants.Counting_Image_MTurk:
            case TaskConstants.Counting_Video:
            case TaskConstants.Counting_Video_MTurk:
                aggResultString = ObjectCountingAggregator.GetAggregatedResultString(resultEntries);
                break;

            case TaskConstants.Detection_Image:
            case TaskConstants.Detection_Image_MTurk:
                aggResultString = MultiObjectLocalizationAndLabelingAggregator.GetAggregatedResultString(resultEntries);
                break;

            case TaskConstants.Tracking:
                aggResultString = null;    //Done testing, marked null so it becomes a demo page.
                break;

            case TaskConstants.Tracking_MTurk:
                aggResultString = MultiObjectTrackingAggregator.GetAggregatedResultString(resultEntries);
                break;

            case TaskConstants.TrackletLabeling:
            case TaskConstants.TrackletLabeling_MTurk:
                aggResultString = TrackletLabelingAggregator.GetAggregatedResultString(resultEntries);
                break;

            case TaskConstants.Segmentation_Image:
                aggResultString = null;    //Done testing, marked null so it becomes a demo page.
                break;

            case TaskConstants.Segmentation_Image_MTurk:
                aggResultString = ImageSegmentationAggregator.GetAggregatedResultString(resultEntries);
                break;

            case TaskConstants.OpenEndedQuestion_Image:
            case TaskConstants.OpenEndedQuestion_Image_MTurk:
                break;
            }
            if (aggResultString != null)
            {
                aggEntry                        = new SatyamAggregatedResultsTableEntry();
                aggEntry.JobGUID                = resultEntries[0].JobGUID;
                aggEntry.JobTemplateType        = resultEntries[0].JobTemplateType;
                aggEntry.SatyamTaskTableEntryID = resultEntries[0].SatyamTaskTableEntryID;
                aggEntry.UserID                 = resultEntries[0].UserID;
                aggEntry.ResultString           = aggResultString;
            }
            return(aggEntry);
        }