public static bool IsGoodWorker(string workerID,
string jobTemplateType,
int minChancesGiven = 5,
double anotherChanceProbablity = 0.2,
double approvalRatioThreshold = 0.5)
{
if (workerID == "")
{
return(true);
}
WorkerStatisticsAccess wsa = new WorkerStatisticsAccess();
WorkerStatisticsTableEntry workerStatistics = wsa.getWorkerStatistics(workerID, jobTemplateType);
wsa.close();
//bool filtered = false;
if (workerStatistics == null)
{
return(true); // first time worker.
}
int noTasksDone = workerStatistics.TasksDone;
if (noTasksDone >= minChancesGiven)
{
double approvalRatio = workerStatistics.SuccessFraction;
if (approvalRatio < approvalRatioThreshold)
{
// roll a dice to decide whether to skip this worker
int rnd = new Random().Next(100);
if (rnd > 100 * anotherChanceProbablity)
{
return(false);
}
else
{
// another chance granted
}
}
}
return(true);
}
//getMaxWeightedGroupByWorkerStatistics
public static ObjectCountingAggregatedResult getAggregatedResultUsingWorkerStatistics(List <SatyamResult> SatyamResults,
int MinResults = TaskConstants.OBJECT_COUNTING_MTURK_MIN_RESULTS_TO_AGGREGATE,
int MaxResults = TaskConstants.OBJECT_COUNTING_MTURK_MAX_RESULTS_TO_AGGREGATE,
double MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND = TaskConstants.OBJECT_COUNTING_MTURK_MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND,
double MAX_DEVIATION_FRACTION = TaskConstants.OBJECT_COUNTING_MTURK_MAX_DEVIATION_FRACTION,
double SUPER_MAJORITY_VALUE = TaskConstants.OBJECT_COUNTING_MTURK_SUPER_MAJORITY_VALUE)
{
if (SatyamResults.Count < MinResults)
{
return(null);
}
Dictionary <int, List <SatyamResult> > resultsCount = new Dictionary <int, List <SatyamResult> >();
Dictionary <double, List <SatyamResult> > clusteredCounts = ClusterCountsSatyamResultByMaxDeviation(SatyamResults, MAX_DEVIATION_FRACTION, MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND, out resultsCount);
List <double> mergedCounts = clusteredCounts.Keys.ToList();
double finalCount = -1;
int maxCount = 0;
//now check if there is consensus by super majority in the merged values
if (clusteredCounts.Count == 1) //if there is only one cluster we are done!!
{
finalCount = mergedCounts[0];
maxCount = clusteredCounts[finalCount].Count;
}
else
{
////find the one with the largest worker success rate, if one of the group has no statistics, just fall back to use maxCounts
maxCount = -1;
int index = 0;
double maxSuccessRate = -1;
int maxSuccRateIndex = 0;
WorkerStatisticsAccess wsa = new WorkerStatisticsAccess();
bool useWorkerStatistics = true;
for (int i = 0; i < clusteredCounts.Count; i++)
{
// using count
if (maxCount < clusteredCounts[mergedCounts[i]].Count)
{
maxCount = clusteredCounts[mergedCounts[i]].Count;
index = i;
}
// using succ rate
//if (!useWorkerStatistics) continue;
int ResultHasWorkerStatistics = 0;
double successRate = 1;
for (int j = 0; j < clusteredCounts[mergedCounts[i]].Count; j++)
{
WorkerStatisticsTableEntry stats = wsa.getWorkerStatistics(clusteredCounts[mergedCounts[i]][j].amazonInfo.WorkerID, TaskConstants.Counting_Image_MTurk);
if (stats != null)
{
successRate *= (1 - stats.SuccessFraction);
}
else
{
successRate *= 0.93;
}
ResultHasWorkerStatistics++;
}
//if (ResultHasWorkerStatistics == 0)
//{
// useWorkerStatistics = false;
// continue;
//}
//successRate = Math.Pow(successRate, 1/(double)ResultHasWorkerStatistics);
successRate = 1 - successRate;
if (maxSuccessRate < successRate)
{
maxSuccessRate = successRate;
maxSuccRateIndex = i;
}
}
wsa.close();
if (useWorkerStatistics)
{
finalCount = mergedCounts[maxSuccRateIndex];
}
else
{
finalCount = mergedCounts[index];
Console.WriteLine("Not using statistics");
}
}
if (maxCount < SatyamResults.Count * SUPER_MAJORITY_VALUE && SatyamResults.Count < MaxResults) //there was no consensus
{
return(null);
}
ObjectCountingAggregatedResult aggresult = new ObjectCountingAggregatedResult();
ObjectCountingAggregatedResultMetaData meta = new ObjectCountingAggregatedResultMetaData();
meta.TotalCount = SatyamResults.Count;
meta.CountsHistogram = new Dictionary <string, int>();
foreach (KeyValuePair <int, List <SatyamResult> > entry in resultsCount)
{
meta.CountsHistogram.Add(entry.Key.ToString(), entry.Value.Count);
}
aggresult.Count = finalCount;
aggresult.metaData = meta;
return(aggresult);
}