public static bool IsGoodWorker(string workerID,
                                        string jobTemplateType,
                                        int minChancesGiven            = 5,
                                        double anotherChanceProbablity = 0.2,
                                        double approvalRatioThreshold  = 0.5)
        {
            if (workerID == "")
            {
                return(true);
            }
            WorkerStatisticsAccess     wsa = new WorkerStatisticsAccess();
            WorkerStatisticsTableEntry workerStatistics = wsa.getWorkerStatistics(workerID, jobTemplateType);

            wsa.close();

            //bool filtered = false;

            if (workerStatistics == null)
            {
                return(true);                         // first time worker.
            }
            int noTasksDone = workerStatistics.TasksDone;

            if (noTasksDone >= minChancesGiven)
            {
                double approvalRatio = workerStatistics.SuccessFraction;

                if (approvalRatio < approvalRatioThreshold)
                {
                    // roll a dice to decide whether to skip this worker
                    int rnd = new Random().Next(100);
                    if (rnd > 100 * anotherChanceProbablity)
                    {
                        return(false);
                    }
                    else
                    {
                        // another chance granted
                    }
                }
            }
            return(true);
        }
Exemple #2
0
        //getMaxWeightedGroupByWorkerStatistics
        public static ObjectCountingAggregatedResult getAggregatedResultUsingWorkerStatistics(List <SatyamResult> SatyamResults,
                                                                                              int MinResults = TaskConstants.OBJECT_COUNTING_MTURK_MIN_RESULTS_TO_AGGREGATE,
                                                                                              int MaxResults = TaskConstants.OBJECT_COUNTING_MTURK_MAX_RESULTS_TO_AGGREGATE,
                                                                                              double MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND = TaskConstants.OBJECT_COUNTING_MTURK_MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND,
                                                                                              double MAX_DEVIATION_FRACTION = TaskConstants.OBJECT_COUNTING_MTURK_MAX_DEVIATION_FRACTION,
                                                                                              double SUPER_MAJORITY_VALUE   = TaskConstants.OBJECT_COUNTING_MTURK_SUPER_MAJORITY_VALUE)
        {
            if (SatyamResults.Count < MinResults)
            {
                return(null);
            }

            Dictionary <int, List <SatyamResult> >    resultsCount    = new Dictionary <int, List <SatyamResult> >();
            Dictionary <double, List <SatyamResult> > clusteredCounts = ClusterCountsSatyamResultByMaxDeviation(SatyamResults, MAX_DEVIATION_FRACTION, MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND, out resultsCount);


            List <double> mergedCounts = clusteredCounts.Keys.ToList();
            double        finalCount   = -1;
            int           maxCount     = 0;

            //now check if there is consensus by super majority in the merged values
            if (clusteredCounts.Count == 1) //if there is only one cluster we are done!!
            {
                finalCount = mergedCounts[0];
                maxCount   = clusteredCounts[finalCount].Count;
            }
            else
            {
                ////find the one with the largest worker success rate, if one of the group has no statistics, just fall back to use maxCounts
                maxCount = -1;
                int    index               = 0;
                double maxSuccessRate      = -1;
                int    maxSuccRateIndex    = 0;
                WorkerStatisticsAccess wsa = new WorkerStatisticsAccess();
                bool useWorkerStatistics   = true;
                for (int i = 0; i < clusteredCounts.Count; i++)
                {
                    // using count
                    if (maxCount < clusteredCounts[mergedCounts[i]].Count)
                    {
                        maxCount = clusteredCounts[mergedCounts[i]].Count;
                        index    = i;
                    }

                    // using succ rate
                    //if (!useWorkerStatistics) continue;

                    int    ResultHasWorkerStatistics = 0;
                    double successRate = 1;
                    for (int j = 0; j < clusteredCounts[mergedCounts[i]].Count; j++)
                    {
                        WorkerStatisticsTableEntry stats = wsa.getWorkerStatistics(clusteredCounts[mergedCounts[i]][j].amazonInfo.WorkerID, TaskConstants.Counting_Image_MTurk);
                        if (stats != null)
                        {
                            successRate *= (1 - stats.SuccessFraction);
                        }
                        else
                        {
                            successRate *= 0.93;
                        }
                        ResultHasWorkerStatistics++;
                    }

                    //if (ResultHasWorkerStatistics == 0)
                    //{
                    //    useWorkerStatistics = false;
                    //    continue;
                    //}

                    //successRate = Math.Pow(successRate, 1/(double)ResultHasWorkerStatistics);
                    successRate = 1 - successRate;

                    if (maxSuccessRate < successRate)
                    {
                        maxSuccessRate   = successRate;
                        maxSuccRateIndex = i;
                    }
                }

                wsa.close();


                if (useWorkerStatistics)
                {
                    finalCount = mergedCounts[maxSuccRateIndex];
                }
                else
                {
                    finalCount = mergedCounts[index];
                    Console.WriteLine("Not using statistics");
                }
            }

            if (maxCount < SatyamResults.Count * SUPER_MAJORITY_VALUE && SatyamResults.Count < MaxResults) //there was no consensus
            {
                return(null);
            }

            ObjectCountingAggregatedResult aggresult = new ObjectCountingAggregatedResult();

            ObjectCountingAggregatedResultMetaData meta = new ObjectCountingAggregatedResultMetaData();

            meta.TotalCount      = SatyamResults.Count;
            meta.CountsHistogram = new Dictionary <string, int>();
            foreach (KeyValuePair <int, List <SatyamResult> > entry in resultsCount)
            {
                meta.CountsHistogram.Add(entry.Key.ToString(), entry.Value.Count);
            }

            aggresult.Count    = finalCount;
            aggresult.metaData = meta;
            return(aggresult);
        }