コード例 #1
0
        //getMaxWeightedGroupByWorkerStatistics
        public static ObjectCountingAggregatedResult getAggregatedResultUsingWorkerStatistics(List <SatyamResult> SatyamResults,
                                                                                              int MinResults = TaskConstants.OBJECT_COUNTING_MTURK_MIN_RESULTS_TO_AGGREGATE,
                                                                                              int MaxResults = TaskConstants.OBJECT_COUNTING_MTURK_MAX_RESULTS_TO_AGGREGATE,
                                                                                              double MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND = TaskConstants.OBJECT_COUNTING_MTURK_MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND,
                                                                                              double MAX_DEVIATION_FRACTION = TaskConstants.OBJECT_COUNTING_MTURK_MAX_DEVIATION_FRACTION,
                                                                                              double SUPER_MAJORITY_VALUE   = TaskConstants.OBJECT_COUNTING_MTURK_SUPER_MAJORITY_VALUE)
        {
            if (SatyamResults.Count < MinResults)
            {
                return(null);
            }

            Dictionary <int, List <SatyamResult> >    resultsCount    = new Dictionary <int, List <SatyamResult> >();
            Dictionary <double, List <SatyamResult> > clusteredCounts = ClusterCountsSatyamResultByMaxDeviation(SatyamResults, MAX_DEVIATION_FRACTION, MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND, out resultsCount);


            List <double> mergedCounts = clusteredCounts.Keys.ToList();
            double        finalCount   = -1;
            int           maxCount     = 0;

            //now check if there is consensus by super majority in the merged values
            if (clusteredCounts.Count == 1) //if there is only one cluster we are done!!
            {
                finalCount = mergedCounts[0];
                maxCount   = clusteredCounts[finalCount].Count;
            }
            else
            {
                ////find the one with the largest worker success rate, if one of the group has no statistics, just fall back to use maxCounts
                maxCount = -1;
                int    index               = 0;
                double maxSuccessRate      = -1;
                int    maxSuccRateIndex    = 0;
                WorkerStatisticsAccess wsa = new WorkerStatisticsAccess();
                bool useWorkerStatistics   = true;
                for (int i = 0; i < clusteredCounts.Count; i++)
                {
                    // using count
                    if (maxCount < clusteredCounts[mergedCounts[i]].Count)
                    {
                        maxCount = clusteredCounts[mergedCounts[i]].Count;
                        index    = i;
                    }

                    // using succ rate
                    //if (!useWorkerStatistics) continue;

                    int    ResultHasWorkerStatistics = 0;
                    double successRate = 1;
                    for (int j = 0; j < clusteredCounts[mergedCounts[i]].Count; j++)
                    {
                        WorkerStatisticsTableEntry stats = wsa.getWorkerStatistics(clusteredCounts[mergedCounts[i]][j].amazonInfo.WorkerID, TaskConstants.Counting_Image_MTurk);
                        if (stats != null)
                        {
                            successRate *= (1 - stats.SuccessFraction);
                        }
                        else
                        {
                            successRate *= 0.93;
                        }
                        ResultHasWorkerStatistics++;
                    }

                    //if (ResultHasWorkerStatistics == 0)
                    //{
                    //    useWorkerStatistics = false;
                    //    continue;
                    //}

                    //successRate = Math.Pow(successRate, 1/(double)ResultHasWorkerStatistics);
                    successRate = 1 - successRate;

                    if (maxSuccessRate < successRate)
                    {
                        maxSuccessRate   = successRate;
                        maxSuccRateIndex = i;
                    }
                }

                wsa.close();


                if (useWorkerStatistics)
                {
                    finalCount = mergedCounts[maxSuccRateIndex];
                }
                else
                {
                    finalCount = mergedCounts[index];
                    Console.WriteLine("Not using statistics");
                }
            }

            if (maxCount < SatyamResults.Count * SUPER_MAJORITY_VALUE && SatyamResults.Count < MaxResults) //there was no consensus
            {
                return(null);
            }

            ObjectCountingAggregatedResult aggresult = new ObjectCountingAggregatedResult();

            ObjectCountingAggregatedResultMetaData meta = new ObjectCountingAggregatedResultMetaData();

            meta.TotalCount      = SatyamResults.Count;
            meta.CountsHistogram = new Dictionary <string, int>();
            foreach (KeyValuePair <int, List <SatyamResult> > entry in resultsCount)
            {
                meta.CountsHistogram.Add(entry.Key.ToString(), entry.Value.Count);
            }

            aggresult.Count    = finalCount;
            aggresult.metaData = meta;
            return(aggresult);
        }
コード例 #2
0
        public static ObjectCountingAggregatedResult getAggregatedResult(List <ObjectCountingResult> results,
                                                                         int MinResults = TaskConstants.OBJECT_COUNTING_MTURK_MIN_RESULTS_TO_AGGREGATE,
                                                                         int MaxResults = TaskConstants.OBJECT_COUNTING_MTURK_MAX_RESULTS_TO_AGGREGATE,
                                                                         double MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND = TaskConstants.OBJECT_COUNTING_MTURK_MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND,
                                                                         double MAX_DEVIATION_FRACTION = TaskConstants.OBJECT_COUNTING_MTURK_MAX_DEVIATION_FRACTION,
                                                                         double SUPER_MAJORITY_VALUE   = TaskConstants.OBJECT_COUNTING_MTURK_SUPER_MAJORITY_VALUE)
        {
            if (results.Count < MinResults)
            {
                return(null);
            }


            Dictionary <int, int> resultCounts = new Dictionary <int, int>();



            foreach (ObjectCountingResult result in results)
            {
                if (!resultCounts.ContainsKey(result.Count))
                {
                    resultCounts.Add(result.Count, 0);
                }
                resultCounts[result.Count]++;
            }

            //First hierachically cluster the counts

            Dictionary <double, int> clusteredCounts = ClusterCountsByMaxDeviation(resultCounts, MAX_DEVIATION_FRACTION, MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND);
            List <double>            mergedCounts    = clusteredCounts.Keys.ToList();
            double finalCount = -1;
            int    maxCount   = 0;

            //now check if there is consensus by super majority in the merged values
            if (clusteredCounts.Count == 1) //if there is only one cluster we are done!!
            {
                finalCount = mergedCounts[0];
                maxCount   = clusteredCounts[finalCount];
            }
            else
            {
                ////find the one with the largest value
                //maxCount = clusteredCounts[mergedCounts[0]];
                int index = 0;
                //for(int i=1;i<clusteredCounts.Count;i++)
                //{
                //    if(maxCount < clusteredCounts[mergedCounts[i]])
                //    {
                //        maxCount = clusteredCounts[mergedCounts[i]];
                //        index = i;
                //    }
                //}

                getMaxGroup(clusteredCounts, out maxCount, out index);
                finalCount = mergedCounts[index];
            }

            if (maxCount < results.Count * SUPER_MAJORITY_VALUE && results.Count < MaxResults) //there was no consensus
            {
                return(null);
            }

            ObjectCountingAggregatedResult aggresult = new ObjectCountingAggregatedResult();

            ObjectCountingAggregatedResultMetaData meta = new ObjectCountingAggregatedResultMetaData();

            meta.TotalCount      = results.Count;
            meta.CountsHistogram = new Dictionary <string, int>();
            foreach (KeyValuePair <int, int> entry in resultCounts)
            {
                meta.CountsHistogram.Add(entry.Key.ToString(), entry.Value);
            }

            aggresult.Count    = finalCount;
            aggresult.metaData = meta;
            return(aggresult);
        }