public static string GetAggregatedResultString(List <SatyamResultsTableEntry> results,
int MinResults = TaskConstants.OBJECT_COUNTING_MTURK_MIN_RESULTS_TO_AGGREGATE,
int MaxResults = TaskConstants.OBJECT_COUNTING_MTURK_MAX_RESULTS_TO_AGGREGATE,
double MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND = TaskConstants.OBJECT_COUNTING_MTURK_MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND,
double MAX_DEVIATION_FRACTION = TaskConstants.OBJECT_COUNTING_MTURK_MAX_DEVIATION_FRACTION,
double SUPER_MAJORITY_VALUE = TaskConstants.OBJECT_COUNTING_MTURK_SUPER_MAJORITY_VALUE)
{
Console.WriteLine("Aggregating");
string resultString = null;
List <ObjectCountingResult> resultList = new List <ObjectCountingResult>();
List <SatyamResult> SatyamResultList = new List <SatyamResult>();
foreach (SatyamResultsTableEntry entry in results)
{
SatyamResult res = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
SatyamResultList.Add(res);
ObjectCountingResult taskr = JSonUtils.ConvertJSonToObject <ObjectCountingResult>(res.TaskResult);
resultList.Add(taskr);
}
//ObjectCountingAggregatedResult r = getAggregatedResult(resultList, MinResults, MaxResults, MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND, MAX_DEVIATION_FRACTION, SUPER_MAJORITY_VALUE);
ObjectCountingAggregatedResult r = getAggregatedResultUsingWorkerStatistics(SatyamResultList, MinResults, MaxResults, MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND, MAX_DEVIATION_FRACTION, SUPER_MAJORITY_VALUE);
if (r != null)
{
string rString = JSonUtils.ConvertObjectToJSon <ObjectCountingAggregatedResult>(r);
SatyamAggregatedResult aggResult = new SatyamAggregatedResult();
aggResult.SatyamTaskTableEntryID = results[0].SatyamTaskTableEntryID;
aggResult.AggregatedResultString = rString;
SatyamResult res = JSonUtils.ConvertJSonToObject <SatyamResult>(results[0].ResultString);
aggResult.TaskParameters = res.TaskParametersString;
resultString = JSonUtils.ConvertObjectToJSon <SatyamAggregatedResult>(aggResult);
}
return(resultString);
}
//public static string GetAggregatedResultString(List<SatyamResultsTableEntry> results,
// int MinResults = TaskConstants.OBJECT_COUNTING_MTURK_MIN_RESULTS_TO_AGGREGATE,
// int MaxResults = TaskConstants.OBJECT_COUNTING_MTURK_MAX_RESULTS_TO_AGGREGATE,
// double MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND = TaskConstants.OBJECT_COUNTING_MTURK_MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND,
// double MAX_DEVIATION_FRACTION = TaskConstants.OBJECT_COUNTING_MTURK_MAX_DEVIATION_FRACTION,
// double SUPER_MAJORITY_VALUE = TaskConstants.OBJECT_COUNTING_MTURK_SUPER_MAJORITY_VALUE)
//{
// string resultString = null;
// List<ObjectCountingResult> resultList = new List<ObjectCountingResult>();
// foreach (SatyamResultsTableEntry entry in results)
// {
// SatyamResult res = JSonUtils.ConvertJSonToObject<SatyamResult>(entry.ResultString);
// ObjectCountingResult taskr = JSonUtils.ConvertJSonToObject<ObjectCountingResult>(res.TaskResult);
// resultList.Add(taskr);
// }
// ObjectCountingAggregatedResult r = getAggregatedResult(resultList, MinResults, MaxResults, MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND, MAX_DEVIATION_FRACTION, SUPER_MAJORITY_VALUE);
// if (r != null)
// {
// string rString = JSonUtils.ConvertObjectToJSon<ObjectCountingAggregatedResult>(r);
// SatyamAggregatedResult aggResult = new SatyamAggregatedResult();
// aggResult.SatyamTaskTableEntryID = results[0].SatyamTaskTableEntryID;
// aggResult.AggregatedResultString = rString;
// SatyamResult res = JSonUtils.ConvertJSonToObject<SatyamResult>(results[0].ResultString);
// aggResult.TaskParameters = res.TaskParametersString;
// resultString = JSonUtils.ConvertObjectToJSon<SatyamAggregatedResult>(aggResult);
// }
// return resultString;
//}
public static bool IsAcceptable(SatyamAggregatedResultsTableEntry aggResult, SatyamResultsTableEntry result,
double MAX_ABSOLUTE_COUNT_DEVIATION = TaskConstants.OBJECT_COUNTING_MTURK_MAX_ABSOLUTE_COUNT_DEVIATION_FOR_PAYMENT,
double MAX_DEVIATION_FRACTION = TaskConstants.OBJECT_COUNTING_MTURK_MAX_DEVIATION_FRACTION_FOR_PAYMENT)
{
SatyamResult res = JSonUtils.ConvertJSonToObject <SatyamResult>(result.ResultString);
SatyamAggregatedResult aggres = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(aggResult.ResultString);
ObjectCountingResult r = JSonUtils.ConvertJSonToObject <ObjectCountingResult>(res.TaskResult);
ObjectCountingAggregatedResult ragg = JSonUtils.ConvertJSonToObject <ObjectCountingAggregatedResult>(aggres.AggregatedResultString);
double maxdev = MAX_ABSOLUTE_COUNT_DEVIATION;
if (ragg.Count * MAX_DEVIATION_FRACTION > maxdev)
{
maxdev = ragg.Count * MAX_DEVIATION_FRACTION;
}
if (Math.Abs(ragg.Count - r.Count) <= maxdev)
{
return(true);
}
else
{
return(false);
}
}
//getMaxWeightedGroupByWorkerStatistics
public static ObjectCountingAggregatedResult getAggregatedResultUsingWorkerStatistics(List <SatyamResult> SatyamResults,
int MinResults = TaskConstants.OBJECT_COUNTING_MTURK_MIN_RESULTS_TO_AGGREGATE,
int MaxResults = TaskConstants.OBJECT_COUNTING_MTURK_MAX_RESULTS_TO_AGGREGATE,
double MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND = TaskConstants.OBJECT_COUNTING_MTURK_MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND,
double MAX_DEVIATION_FRACTION = TaskConstants.OBJECT_COUNTING_MTURK_MAX_DEVIATION_FRACTION,
double SUPER_MAJORITY_VALUE = TaskConstants.OBJECT_COUNTING_MTURK_SUPER_MAJORITY_VALUE)
{
if (SatyamResults.Count < MinResults)
{
return(null);
}
Dictionary <int, List <SatyamResult> > resultsCount = new Dictionary <int, List <SatyamResult> >();
Dictionary <double, List <SatyamResult> > clusteredCounts = ClusterCountsSatyamResultByMaxDeviation(SatyamResults, MAX_DEVIATION_FRACTION, MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND, out resultsCount);
List <double> mergedCounts = clusteredCounts.Keys.ToList();
double finalCount = -1;
int maxCount = 0;
//now check if there is consensus by super majority in the merged values
if (clusteredCounts.Count == 1) //if there is only one cluster we are done!!
{
finalCount = mergedCounts[0];
maxCount = clusteredCounts[finalCount].Count;
}
else
{
////find the one with the largest worker success rate, if one of the group has no statistics, just fall back to use maxCounts
maxCount = -1;
int index = 0;
double maxSuccessRate = -1;
int maxSuccRateIndex = 0;
WorkerStatisticsAccess wsa = new WorkerStatisticsAccess();
bool useWorkerStatistics = true;
for (int i = 0; i < clusteredCounts.Count; i++)
{
// using count
if (maxCount < clusteredCounts[mergedCounts[i]].Count)
{
maxCount = clusteredCounts[mergedCounts[i]].Count;
index = i;
}
// using succ rate
//if (!useWorkerStatistics) continue;
int ResultHasWorkerStatistics = 0;
double successRate = 1;
for (int j = 0; j < clusteredCounts[mergedCounts[i]].Count; j++)
{
WorkerStatisticsTableEntry stats = wsa.getWorkerStatistics(clusteredCounts[mergedCounts[i]][j].amazonInfo.WorkerID, TaskConstants.Counting_Image_MTurk);
if (stats != null)
{
successRate *= (1 - stats.SuccessFraction);
}
else
{
successRate *= 0.93;
}
ResultHasWorkerStatistics++;
}
//if (ResultHasWorkerStatistics == 0)
//{
// useWorkerStatistics = false;
// continue;
//}
//successRate = Math.Pow(successRate, 1/(double)ResultHasWorkerStatistics);
successRate = 1 - successRate;
if (maxSuccessRate < successRate)
{
maxSuccessRate = successRate;
maxSuccRateIndex = i;
}
}
wsa.close();
if (useWorkerStatistics)
{
finalCount = mergedCounts[maxSuccRateIndex];
}
else
{
finalCount = mergedCounts[index];
Console.WriteLine("Not using statistics");
}
}
if (maxCount < SatyamResults.Count * SUPER_MAJORITY_VALUE && SatyamResults.Count < MaxResults) //there was no consensus
{
return(null);
}
ObjectCountingAggregatedResult aggresult = new ObjectCountingAggregatedResult();
ObjectCountingAggregatedResultMetaData meta = new ObjectCountingAggregatedResultMetaData();
meta.TotalCount = SatyamResults.Count;
meta.CountsHistogram = new Dictionary <string, int>();
foreach (KeyValuePair <int, List <SatyamResult> > entry in resultsCount)
{
meta.CountsHistogram.Add(entry.Key.ToString(), entry.Value.Count);
}
aggresult.Count = finalCount;
aggresult.metaData = meta;
return(aggresult);
}
public static ObjectCountingAggregatedResult getAggregatedResult(List <ObjectCountingResult> results,
int MinResults = TaskConstants.OBJECT_COUNTING_MTURK_MIN_RESULTS_TO_AGGREGATE,
int MaxResults = TaskConstants.OBJECT_COUNTING_MTURK_MAX_RESULTS_TO_AGGREGATE,
double MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND = TaskConstants.OBJECT_COUNTING_MTURK_MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND,
double MAX_DEVIATION_FRACTION = TaskConstants.OBJECT_COUNTING_MTURK_MAX_DEVIATION_FRACTION,
double SUPER_MAJORITY_VALUE = TaskConstants.OBJECT_COUNTING_MTURK_SUPER_MAJORITY_VALUE)
{
if (results.Count < MinResults)
{
return(null);
}
Dictionary <int, int> resultCounts = new Dictionary <int, int>();
foreach (ObjectCountingResult result in results)
{
if (!resultCounts.ContainsKey(result.Count))
{
resultCounts.Add(result.Count, 0);
}
resultCounts[result.Count]++;
}
//First hierachically cluster the counts
Dictionary <double, int> clusteredCounts = ClusterCountsByMaxDeviation(resultCounts, MAX_DEVIATION_FRACTION, MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND);
List <double> mergedCounts = clusteredCounts.Keys.ToList();
double finalCount = -1;
int maxCount = 0;
//now check if there is consensus by super majority in the merged values
if (clusteredCounts.Count == 1) //if there is only one cluster we are done!!
{
finalCount = mergedCounts[0];
maxCount = clusteredCounts[finalCount];
}
else
{
////find the one with the largest value
//maxCount = clusteredCounts[mergedCounts[0]];
int index = 0;
//for(int i=1;i<clusteredCounts.Count;i++)
//{
// if(maxCount < clusteredCounts[mergedCounts[i]])
// {
// maxCount = clusteredCounts[mergedCounts[i]];
// index = i;
// }
//}
getMaxGroup(clusteredCounts, out maxCount, out index);
finalCount = mergedCounts[index];
}
if (maxCount < results.Count * SUPER_MAJORITY_VALUE && results.Count < MaxResults) //there was no consensus
{
return(null);
}
ObjectCountingAggregatedResult aggresult = new ObjectCountingAggregatedResult();
ObjectCountingAggregatedResultMetaData meta = new ObjectCountingAggregatedResultMetaData();
meta.TotalCount = results.Count;
meta.CountsHistogram = new Dictionary <string, int>();
foreach (KeyValuePair <int, int> entry in resultCounts)
{
meta.CountsHistogram.Add(entry.Key.ToString(), entry.Value);
}
aggresult.Count = finalCount;
aggresult.metaData = meta;
return(aggresult);
}