//public static string GetAggregatedResultString(List<SatyamResultsTableEntry> results,
// int MinResults = TaskConstants.OBJECT_COUNTING_MTURK_MIN_RESULTS_TO_AGGREGATE,
// int MaxResults = TaskConstants.OBJECT_COUNTING_MTURK_MAX_RESULTS_TO_AGGREGATE,
// double MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND = TaskConstants.OBJECT_COUNTING_MTURK_MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND,
// double MAX_DEVIATION_FRACTION = TaskConstants.OBJECT_COUNTING_MTURK_MAX_DEVIATION_FRACTION,
// double SUPER_MAJORITY_VALUE = TaskConstants.OBJECT_COUNTING_MTURK_SUPER_MAJORITY_VALUE)
//{
// string resultString = null;
// List<ObjectCountingResult> resultList = new List<ObjectCountingResult>();
// foreach (SatyamResultsTableEntry entry in results)
// {
// SatyamResult res = JSonUtils.ConvertJSonToObject<SatyamResult>(entry.ResultString);
// ObjectCountingResult taskr = JSonUtils.ConvertJSonToObject<ObjectCountingResult>(res.TaskResult);
// resultList.Add(taskr);
// }
// ObjectCountingAggregatedResult r = getAggregatedResult(resultList, MinResults, MaxResults, MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND, MAX_DEVIATION_FRACTION, SUPER_MAJORITY_VALUE);
// if (r != null)
// {
// string rString = JSonUtils.ConvertObjectToJSon<ObjectCountingAggregatedResult>(r);
// SatyamAggregatedResult aggResult = new SatyamAggregatedResult();
// aggResult.SatyamTaskTableEntryID = results[0].SatyamTaskTableEntryID;
// aggResult.AggregatedResultString = rString;
// SatyamResult res = JSonUtils.ConvertJSonToObject<SatyamResult>(results[0].ResultString);
// aggResult.TaskParameters = res.TaskParametersString;
// resultString = JSonUtils.ConvertObjectToJSon<SatyamAggregatedResult>(aggResult);
// }
// return resultString;
//}
public static bool IsAcceptable(SatyamAggregatedResultsTableEntry aggResult, SatyamResultsTableEntry result,
double MAX_ABSOLUTE_COUNT_DEVIATION = TaskConstants.OBJECT_COUNTING_MTURK_MAX_ABSOLUTE_COUNT_DEVIATION_FOR_PAYMENT,
double MAX_DEVIATION_FRACTION = TaskConstants.OBJECT_COUNTING_MTURK_MAX_DEVIATION_FRACTION_FOR_PAYMENT)
{
SatyamResult res = JSonUtils.ConvertJSonToObject <SatyamResult>(result.ResultString);
SatyamAggregatedResult aggres = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(aggResult.ResultString);
ObjectCountingResult r = JSonUtils.ConvertJSonToObject <ObjectCountingResult>(res.TaskResult);
ObjectCountingAggregatedResult ragg = JSonUtils.ConvertJSonToObject <ObjectCountingAggregatedResult>(aggres.AggregatedResultString);
double maxdev = MAX_ABSOLUTE_COUNT_DEVIATION;
if (ragg.Count * MAX_DEVIATION_FRACTION > maxdev)
{
maxdev = ragg.Count * MAX_DEVIATION_FRACTION;
}
if (Math.Abs(ragg.Count - r.Count) <= maxdev)
{
return(true);
}
else
{
return(false);
}
}
public static string GetAggregatedResultString(List <SatyamResultsTableEntry> results,
int MinResults = TaskConstants.OBJECT_COUNTING_MTURK_MIN_RESULTS_TO_AGGREGATE,
int MaxResults = TaskConstants.OBJECT_COUNTING_MTURK_MAX_RESULTS_TO_AGGREGATE,
double MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND = TaskConstants.OBJECT_COUNTING_MTURK_MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND,
double MAX_DEVIATION_FRACTION = TaskConstants.OBJECT_COUNTING_MTURK_MAX_DEVIATION_FRACTION,
double SUPER_MAJORITY_VALUE = TaskConstants.OBJECT_COUNTING_MTURK_SUPER_MAJORITY_VALUE)
{
Console.WriteLine("Aggregating");
string resultString = null;
List <ObjectCountingResult> resultList = new List <ObjectCountingResult>();
List <SatyamResult> SatyamResultList = new List <SatyamResult>();
foreach (SatyamResultsTableEntry entry in results)
{
SatyamResult res = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
SatyamResultList.Add(res);
ObjectCountingResult taskr = JSonUtils.ConvertJSonToObject <ObjectCountingResult>(res.TaskResult);
resultList.Add(taskr);
}
//ObjectCountingAggregatedResult r = getAggregatedResult(resultList, MinResults, MaxResults, MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND, MAX_DEVIATION_FRACTION, SUPER_MAJORITY_VALUE);
ObjectCountingAggregatedResult r = getAggregatedResultUsingWorkerStatistics(SatyamResultList, MinResults, MaxResults, MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND, MAX_DEVIATION_FRACTION, SUPER_MAJORITY_VALUE);
if (r != null)
{
string rString = JSonUtils.ConvertObjectToJSon <ObjectCountingAggregatedResult>(r);
SatyamAggregatedResult aggResult = new SatyamAggregatedResult();
aggResult.SatyamTaskTableEntryID = results[0].SatyamTaskTableEntryID;
aggResult.AggregatedResultString = rString;
SatyamResult res = JSonUtils.ConvertJSonToObject <SatyamResult>(results[0].ResultString);
aggResult.TaskParameters = res.TaskParametersString;
resultString = JSonUtils.ConvertObjectToJSon <SatyamAggregatedResult>(aggResult);
}
return(resultString);
}
protected void SubmitButton_Click(object sender, EventArgs e)
{
DateTime SubmitTime = DateTime.Now;
DateTime PageLoadTime = Convert.ToDateTime(Hidden_PageLoadTime.Value);
int count;
bool isValidCount = int.TryParse(CountTextBox.Text, out count);
if (!isValidCount)
{
ErrorLabel.Text = "Error : Ivalid Count.";
ErrorLabel.ForeColor = System.Drawing.Color.Red;
ErrorLabel.Font.Bold = true;
}
else if (count < 0)
{
ErrorLabel.Text = "Error : Count cannot be less than zero.";
ErrorLabel.ForeColor = System.Drawing.Color.Red;
ErrorLabel.Font.Bold = true;
}
else
{
ErrorLabel.Text = "";
SatyamTaskTableEntry taskEntry = JSonUtils.ConvertJSonToObject <SatyamTaskTableEntry>(Hidden_TaskEntryString.Value);
SatyamResult result = new SatyamResult();
result.TaskParametersString = taskEntry.TaskParametersString;
result.TaskStartTime = PageLoadTime;
result.TaskEndTime = SubmitTime;
result.TaskTableEntryID = taskEntry.ID;
AmazonTaskResultInfo amazonInfo = new AmazonTaskResultInfo();
amazonInfo.AssignmentID = "";
amazonInfo.WorkerID = "";
amazonInfo.HITID = "";
result.amazonInfo = amazonInfo;
ObjectCountingResult sresult = new ObjectCountingResult();
sresult.Count = count;
string sresultString = JSonUtils.ConvertObjectToJSon <ObjectCountingResult>(sresult);
result.TaskResult = sresultString;
string resultString = JSonUtils.ConvertObjectToJSon <SatyamResult>(result);
SatyamResultsTableAccess resultdb = new SatyamResultsTableAccess();
resultdb.AddEntry(taskEntry.JobTemplateType, taskEntry.UserID, taskEntry.JobGUID, resultString, taskEntry.ID, PageLoadTime, SubmitTime);
resultdb.close();
//SatyamTaskTableAccess taskDB = new SatyamTaskTableAccess();
//taskDB.IncrementDoneScore(taskEntry.ID);
bool NotDone = getNewRandomJob();
if (NotDone == false)
{
Response.Redirect("AllJobsDone.aspx");
}
}
}
protected void SubmitButton_Click(object sender, EventArgs e)
{
DateTime SubmitTime = DateTime.Now;
DateTime PageLoadTime = Convert.ToDateTime(Hidden_PageLoadTime.Value);
int count;
bool isValidCount = int.TryParse(CountTextBox.Text, out count);
if (!isValidCount)
{
ErrorLabel.Text = "Error : Ivalid Count.";
ErrorLabel.ForeColor = System.Drawing.Color.Red;
ErrorLabel.Font.Bold = true;
}
else if (count < 0)
{
ErrorLabel.Text = "Error : Count cannot be less than zero.";
ErrorLabel.ForeColor = System.Drawing.Color.Red;
ErrorLabel.Font.Bold = true;
}
else
{
ErrorLabel.Text = "";
CountTextBox.Text = "";
SatyamTaskTableEntry taskEntry = JSonUtils.ConvertJSonToObject <SatyamTaskTableEntry>(Hidden_TaskEntryString.Value);
SatyamResult result = new SatyamResult();
result.TaskParametersString = taskEntry.TaskParametersString;
result.TaskStartTime = PageLoadTime;
result.TaskEndTime = SubmitTime;
result.TaskTableEntryID = taskEntry.ID;
AmazonTaskResultInfo amazonInfo = new AmazonTaskResultInfo();
amazonInfo.AssignmentID = Hidden_AmazonAssignmentID.Value;
amazonInfo.WorkerID = Hidden_AmazonWorkerID.Value;
amazonInfo.HITID = Hidden_HITID.Value;
amazonInfo.PricePerHIT = Convert.ToDouble(Hidden_Price.Value);
result.amazonInfo = amazonInfo;
ObjectCountingResult sresult = new ObjectCountingResult();
sresult.Count = count;
string sresultString = JSonUtils.ConvertObjectToJSon <ObjectCountingResult>(sresult);
result.TaskResult = sresultString;
string resultString = JSonUtils.ConvertObjectToJSon <SatyamResult>(result);
SatyamResultsTableAccess resultdb = new SatyamResultsTableAccess();
resultdb.AddEntry(taskEntry.JobTemplateType, taskEntry.UserID, taskEntry.JobGUID, resultString, taskEntry.ID, PageLoadTime, SubmitTime);
resultdb.close();
SatyamTaskTableManagement.UpdateResultNumber(taskEntry.ID);
//SatyamTaskTableAccess taskDB = new SatyamTaskTableAccess();
//taskDB.IncrementDoneScore(taskEntry.ID);
int noDone = Convert.ToInt32(Hidden_NoImagesDone.Value);
noDone++;
Hidden_NoImagesDone.Value = noDone.ToString();
bool NotDone = false;
int noTasksPerJob = Convert.ToInt32(Hidden_TasksPerJob.Value);
if (noDone < noTasksPerJob)
{
NotDone = getNewRandomJob();
}
if (NotDone == false)
{
if (!Testing)
{
SatyamAmazonHITTableAccess HITdb = new SatyamAmazonHITTableAccess();
string HITID = result.amazonInfo.HITID;
HITdb.UpdateStatusByHITID(HITID, HitStatus.submitted);
HITdb.close();
AmazonMTurkNotification.submitAmazonTurkHit(result.amazonInfo.AssignmentID, result.amazonInfo.WorkerID, false);
}
Response.Redirect("MTurkTaskDonePage.aspx");
}
}
}
private static Dictionary <double, List <SatyamResult> > ClusterCountsSatyamResultByMaxDeviation(List <SatyamResult> satyamResults, double MAX_DEVIATION_FRACTION, double MAX_ABSOLUTE_COUNT_DEVIATION, out Dictionary <int, List <SatyamResult> > resultsCount)
{
resultsCount = new Dictionary <int, List <SatyamResult> >();
foreach (SatyamResult r in satyamResults)
{
ObjectCountingResult result = JSonUtils.ConvertJSonToObject <ObjectCountingResult>(r.TaskResult);
if (!resultsCount.ContainsKey(result.Count))
{
resultsCount.Add(result.Count, new List <SatyamResult>());
}
resultsCount[result.Count].Add(r);
}
List <int> countsList = resultsCount.Keys.ToList();
//Dictionary<double, int> clusterHeads = new Dictionary<double, int>();
List <KeyValuePair <double, List <SatyamResult> > > clusterHeads = new List <KeyValuePair <double, List <SatyamResult> > >();
//initialize clusterheads with same entries
foreach (KeyValuePair <int, List <SatyamResult> > entry in resultsCount)
{
clusterHeads.Add(new KeyValuePair <double, List <SatyamResult> >(entry.Key, entry.Value));
}
//first cluster them hierchically
bool merged = false;
do
{
List <double> clusterValues = new List <double>();
foreach (KeyValuePair <double, List <SatyamResult> > ch in clusterHeads)
{
clusterValues.Add(ch.Key);
}
//List<double> clusterValues = clusterHeads.Keys.ToList();
//first find the closest numbers with the max counts
int index1 = -1, index2 = -1;
int maxCount = -1;
double bestMergedValue = -1;
merged = false;
for (int i = 0; i < clusterValues.Count; i++)
{
for (int j = i + 1; j < clusterValues.Count; j++)
{
//merged value is the weighted average
double mergedValue = (clusterValues[i] * clusterHeads[i].Value.Count + clusterValues[j] * clusterHeads[j].Value.Count) / (clusterHeads[i].Value.Count + clusterHeads[j].Value.Count);
//find the maximum possible deviation
double maxdev = MAX_ABSOLUTE_COUNT_DEVIATION;
if (Math.Max(clusterValues[i], clusterValues[j]) * MAX_DEVIATION_FRACTION > maxdev)
{
maxdev = Math.Max(clusterValues[i], clusterValues[j]) * MAX_DEVIATION_FRACTION;
}
//check if they match the deviation criterion
if (Math.Abs(clusterValues[i] - clusterValues[j]) <= maxdev)
{
int mergedCount = clusterHeads[i].Value.Count + clusterHeads[j].Value.Count;
if (mergedCount > maxCount)
{
index1 = i;
index2 = j;
maxCount = mergedCount;
merged = true;
bestMergedValue = mergedValue;
}
}
}
}
if (merged) //there are mergable clusterheads
{
//add the merged cluster head
List <SatyamResult> mergedResults = new List <SatyamResult>();
foreach (SatyamResult r in clusterHeads[index1].Value)
{
mergedResults.Add(r);
}
foreach (SatyamResult r in clusterHeads[index2].Value)
{
mergedResults.Add(r);
}
clusterHeads.Add(new KeyValuePair <double, List <SatyamResult> >(bestMergedValue, mergedResults));
//remove the orignial clusterheads
clusterHeads.RemoveAt(index1);
clusterHeads.RemoveAt(index2 - 1); // index 2 is always larger than index 1
}
} while (merged);
Dictionary <double, List <SatyamResult> > aggregatedClusterHeads = new Dictionary <double, List <SatyamResult> >();
for (int i = 0; i < clusterHeads.Count; i++)
{
aggregatedClusterHeads.Add(clusterHeads[i].Key, clusterHeads[i].Value);
}
return(aggregatedClusterHeads);
}
