//public static void AnalyzeApprovalRateByGUID(List<string> guids)
//{
// Console.WriteLine("Analyzing Results Statistics of {0}", guids[0]);
// SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
// List<SatyamResultsTableEntry> entries = new List<SatyamResultsTableEntry>();
// foreach (string guid in guids)
// {
// entries.AddRange(resultsDB.getEntriesByGUIDOrderByID(guid));
// }
// resultsDB.close();
//}
public static void AggregationAnalysis(List <SatyamAggregatedResultsTableEntry> aggEntries,
//List<SatyamResultsTableEntry> ResultEntries,
SortedDictionary <DateTime, List <SatyamResultsTableEntry> > entriesBySubmitTime,
Dictionary <int, int> noResultsNeededForAggregation,
Dictionary <int, int> noResultsNeededForAggregation_new,
string guid, string configString,
int minChancesGiven = 3,
double anotherChanceProbablity = 0,
double approvalRatioThreshold = 0.5,
bool usePreApprovalResult = false)
{
if (noResultsNeededForAggregation == null)
{
noResultsNeededForAggregation = new Dictionary <int, int>();
}
if (noResultsNeededForAggregation_new == null)
{
noResultsNeededForAggregation_new = new Dictionary <int, int>();
}
string approvalString = configString +
"_Thresh_" + approvalRatioThreshold +
"_Explore_" + anotherChanceProbablity;
Dictionary <int, SatyamAggregatedResultsTableEntry> aggEntriesByTaskID = new Dictionary <int, SatyamAggregatedResultsTableEntry>();
Dictionary <int, List <SatyamResultsTableEntry> > ResultEntriesByTaskID = new Dictionary <int, List <SatyamResultsTableEntry> >();
//Dictionary<DateTime, double> TotalApprovalRatioByTime = new Dictionary<DateTime, double>();
List <string> TotalApprovalRatioByTime = new List <string>();
List <string> TotalApprovalRatioByResult = new List <string>();
List <string> FilteredTotalApprovalRatioByTime = new List <string>();
List <string> FilteredTotalApprovalRatioByResult = new List <string>();
List <SatyamResultsTableEntry> approvedEntries = new List <SatyamResultsTableEntry>();
List <SatyamResultsTableEntry> rejectedEntries = new List <SatyamResultsTableEntry>();
List <SatyamResultsTableEntry> InconclusiveEntries = new List <SatyamResultsTableEntry>();
List <SatyamResultsTableEntry> approvedEntriesActuallyPaid = new List <SatyamResultsTableEntry>();
Dictionary <string, int> noTasksApprovedPerWorker = new Dictionary <string, int>();
Dictionary <string, int> noTasksDonePerWorker = new Dictionary <string, int>();
Dictionary <string, double> ApprovalRatioPerWorker = new Dictionary <string, double>();
Dictionary <string, bool> FirstTimersWorkersRecord = new Dictionary <string, bool>();
List <SatyamResultsTableEntry> FilteredApprovedEntries = new List <SatyamResultsTableEntry>();
List <SatyamResultsTableEntry> filteredRejectedEntries = new List <SatyamResultsTableEntry>();
Dictionary <int, double> AggregationTimePerTask = new Dictionary <int, double>();
foreach (SatyamAggregatedResultsTableEntry aggEntry in aggEntries)
{
if (!aggEntriesByTaskID.ContainsKey(aggEntry.SatyamTaskTableEntryID))
{
aggEntriesByTaskID.Add(aggEntry.SatyamTaskTableEntryID, aggEntry);
}
else
{
// duplicate agg entry for a task
}
}
//Dictionary<int, int> noResultsNeededForAggregation = getNoResultsNeededForAggregationFromLog(approvalString, guid);
foreach (int k in noResultsNeededForAggregation_new.Keys)
{
noResultsNeededForAggregation.Add(k, noResultsNeededForAggregation_new[k]);
}
int count = 0;
foreach (DateTime t in entriesBySubmitTime.Keys)
{
//Console.WriteLine("Processing Results of time: {0}", t);
Console.WriteLine("Judging result: {0}/{1}", count, entriesBySubmitTime.Count);
count++;
List <SatyamResultsTableEntry> ResultEntries = entriesBySubmitTime[t];
foreach (SatyamResultsTableEntry res in ResultEntries)
{
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(res.ResultString);
int taskID = res.SatyamTaskTableEntryID;
if (!ResultEntriesByTaskID.ContainsKey(taskID))
{
ResultEntriesByTaskID.Add(taskID, new List <SatyamResultsTableEntry>());
}
ResultEntriesByTaskID[taskID].Add(res);
if (!aggEntriesByTaskID.ContainsKey(taskID))
{
//foreach (SatyamResultsTableEntry res in ResultEntriesByTaskID[taskID])
//{
//InconclusiveEntries.Add(res);
//}
InconclusiveEntries.Add(res);
continue;
}
SatyamAggregatedResultsTableEntry agg = aggEntriesByTaskID[taskID];
//foreach (SatyamResultsTableEntry res in ResultEntriesByTaskID[taskID])
//{
//SatyamResult satyamResult = JSonUtils.ConvertJSonToObject<SatyamResult>(res.ResultString);
string workerId = satyamResult.amazonInfo.WorkerID;
// the worker pre-filtering step
bool filtered = false;
if (noTasksDonePerWorker.ContainsKey(workerId) && noTasksDonePerWorker[workerId] >= minChancesGiven)
{
double approvalRatio = (double)noTasksApprovedPerWorker[workerId] / (double)noTasksDonePerWorker[workerId];
ApprovalRatioPerWorker[workerId] = approvalRatio;
if (approvalRatio < approvalRatioThreshold)
{
// roll a dice to decide whether to skip this worker
int rnd = new Random().Next(100);
if (rnd > 100 * anotherChanceProbablity)
{
filtered = true;
}
else
{
// another chance granted
}
}
}
if (!noTasksDonePerWorker.ContainsKey(workerId))
{
noTasksDonePerWorker.Add(workerId, 0);
noTasksApprovedPerWorker.Add(workerId, 0);
ApprovalRatioPerWorker.Add(workerId, -1);
}
if (!filtered)
{
noTasksDonePerWorker[workerId]++;
}
bool acceptable = false;
if (usePreApprovalResult)
{
if (ResultStatus.acceptedStatusList.Contains(res.Status))
{
acceptable = true;
}
}
else
{
acceptable = AcceptanceCriterionChecker.IsAcceptable(agg, res);
}
if (acceptable)
{
approvedEntries.Add(res);
if (!FirstTimersWorkersRecord.ContainsKey(workerId))
{
FirstTimersWorkersRecord.Add(workerId, true);
}
if (!filtered)
{
noTasksApprovedPerWorker[workerId]++;
FilteredApprovedEntries.Add(res);
}
if (noResultsNeededForAggregation != null && noResultsNeededForAggregation.ContainsKey(taskID) &&
ResultEntriesByTaskID[taskID].Count <= noResultsNeededForAggregation[taskID])
{
approvedEntriesActuallyPaid.Add(res);
}
}
else
{
rejectedEntries.Add(res);
if (!FirstTimersWorkersRecord.ContainsKey(workerId))
{
FirstTimersWorkersRecord.Add(workerId, false);
}
if (!filtered)
{
filteredRejectedEntries.Add(res);
}
}
//}
/// timetaken till aggregation
if (noResultsNeededForAggregation != null && noResultsNeededForAggregation.ContainsKey(taskID) &&
ResultEntriesByTaskID[taskID].Count == noResultsNeededForAggregation[taskID])
{
// aggregation happen
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
SatyamJob job = task.jobEntry;
if (!AggregationTimePerTask.ContainsKey(taskID))
{
AggregationTimePerTask.Add(taskID, (res.SubmitTime - job.JobSubmitTime).TotalSeconds);
}
}
string ApprovalString = approvedEntries.Count + " " + (approvedEntries.Count + rejectedEntries.Count) + " "
+ (double)approvedEntries.Count / (double)(approvedEntries.Count + rejectedEntries.Count);
TotalApprovalRatioByResult.Add(ApprovalString);
string filteredApprovalString = FilteredApprovedEntries.Count + " " + (FilteredApprovedEntries.Count + filteredRejectedEntries.Count) + " "
+ (double)FilteredApprovedEntries.Count / (double)(FilteredApprovedEntries.Count + filteredRejectedEntries.Count);
FilteredTotalApprovalRatioByResult.Add(filteredApprovalString);
}
string approvalStringByTime = t.ToString() + " " + approvedEntries.Count + " " + (approvedEntries.Count + rejectedEntries.Count) + " "
+ (double)approvedEntries.Count / (double)(approvedEntries.Count + rejectedEntries.Count);
TotalApprovalRatioByTime.Add(approvalStringByTime);
string filteredApprovalStringByTime = t.ToString() + " " + FilteredApprovedEntries.Count + " " + (FilteredApprovedEntries.Count + filteredRejectedEntries.Count) + " "
+ (double)FilteredApprovedEntries.Count / (double)(FilteredApprovedEntries.Count + filteredRejectedEntries.Count);
FilteredTotalApprovalRatioByTime.Add(filteredApprovalStringByTime);
}
// count first timer succ rate
int FirstTimeSucc = 0;
foreach (KeyValuePair <string, bool> wid in FirstTimersWorkersRecord)
{
if (wid.Value)
{
FirstTimeSucc++;
}
}
double FirstTimeSuccRate = (double)FirstTimeSucc / FirstTimersWorkersRecord.Count;
Console.WriteLine("Approval #, ratio: {0},{1}", approvedEntries.Count, (double)approvedEntries.Count / (double)(approvedEntries.Count + rejectedEntries.Count));
Console.WriteLine("Filtered Approval #, ratio: {0},{1}", FilteredApprovedEntries.Count, (double)FilteredApprovedEntries.Count / (double)(FilteredApprovedEntries.Count + filteredRejectedEntries.Count));
int approvePrevRejected = 0;
List <string> ApprovedAssignmentIDs = new List <string>();
foreach (SatyamResultsTableEntry approved in approvedEntries)
{
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(approved.ResultString);
ApprovedAssignmentIDs.Add(satyamResult.amazonInfo.AssignmentID);
if (approved.Status == ResultStatus.accepted_NotPaid || approved.Status == ResultStatus.rejected_NotPaid || approved.Status == ResultStatus.rejected)
{
approvePrevRejected++;
}
}
Console.WriteLine("Approving Previously Rejected: {0}, {1}", approvePrevRejected, (double)approvePrevRejected / (double)approvedEntries.Count);
string outputFileDir = DirectoryConstants.defaultTempDirectory + guid + "\\ApprovalAnalysis\\";
if (guid != null && approvalString != null)
{
if (!Directory.Exists(outputFileDir))
{
Directory.CreateDirectory(outputFileDir);
}
}
string summaryFile = outputFileDir + "resultSummary.txt";
string summary = String.Format("{0} {1} {2} {3} {4} {5} {6} {7}\n",
approvalString,
approvedEntries.Count, (double)approvedEntries.Count / (double)(approvedEntries.Count + rejectedEntries.Count),
approvedEntriesActuallyPaid.Count,
FilteredApprovedEntries.Count, (double)FilteredApprovedEntries.Count / (double)(FilteredApprovedEntries.Count + filteredRejectedEntries.Count),
approvePrevRejected, (double)approvePrevRejected / (double)approvedEntries.Count);
File.AppendAllText(summaryFile, summary);
string outputFile = outputFileDir + approvalString + ".txt";
File.WriteAllLines(outputFile, ApprovedAssignmentIDs.ToArray());
//string outputFilePaid = outputFileDir + approvalString + "_ActuallyPaid.txt";
//File.WriteAllText(outputFilePaid, approvedEntriesActuallyPaid.Count.ToString());
List <string> approvalRates = new List <string>();
foreach (string worker in ApprovalRatioPerWorker.Keys)
{
approvalRates.Add(ApprovalRatioPerWorker[worker].ToString());
}
string approvalDistributionFile = outputFileDir + approvalString + "_dist.txt";
File.WriteAllLines(approvalDistributionFile, approvalRates.ToArray());
string approvalByTimeFile = outputFileDir + approvalString + "_approvalRateByTime.txt";
string approvalByResultFile = outputFileDir + approvalString + "_approvalRateByResults.txt";
File.WriteAllLines(approvalByTimeFile, TotalApprovalRatioByTime.ToArray());
File.WriteAllLines(approvalByResultFile, TotalApprovalRatioByResult.ToArray());
approvalByTimeFile = outputFileDir + approvalString + "_approvalRateByTime_filtered.txt";
approvalByResultFile = outputFileDir + approvalString + "_approvalRateByResults_filtered.txt";
File.WriteAllLines(approvalByTimeFile, FilteredTotalApprovalRatioByTime.ToArray());
File.WriteAllLines(approvalByResultFile, FilteredTotalApprovalRatioByResult.ToArray());
List <string> aggregationTime = new List <string>();
foreach (int task in AggregationTimePerTask.Keys)
{
aggregationTime.Add(task.ToString() + " " + AggregationTimePerTask[task].ToString());
}
string aggregationTimeFile = outputFileDir + approvalString + "_aggregationTime.txt";
File.WriteAllLines(aggregationTimeFile, aggregationTime.ToArray());
}
public static void Run([QueueTrigger("judge-result")] string myQueueItem, TraceWriter log)
{
bool logging = false;
string[] fields = myQueueItem.Split('_');
string guid = fields[0];
int taskID = Convert.ToInt32(fields[1]);
string resultID = fields[2];
if (logging)
{
log.Info($"Judge Result: {myQueueItem}");
}
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
List <SatyamResultsTableEntry> results = resultsDB.getEntriesID(resultID);
resultsDB.close();
if (results.Count == 0)
{
return;
}
SatyamAggregatedResultsTableAccess aggDB = new SatyamAggregatedResultsTableAccess();
SatyamAggregatedResultsTableEntry aggResult = aggDB.getEntryByTaskID(taskID);
aggDB.close();
if (aggResult == null)
{
return;
}
SatyamResultsTableEntry result = results[0];
SatyamResult res = JSonUtils.ConvertJSonToObject <SatyamResult>(result.ResultString);
string taskTemplateType = result.JobTemplateType;
string workerID = res.amazonInfo.WorkerID;
DateTime doneTime = res.TaskEndTime;
if (AcceptanceCriterionChecker.IsAcceptable(aggResult, result))
{
resultsDB = new SatyamResultsTableAccess();
resultsDB.UpdateStatusByID(result.ID, ResultStatus.accepted);
resultsDB.close();
WorkerStatisticsManagement.UpdateWorkerStatistics(workerID, taskTemplateType, true, doneTime);
if (logging)
{
log.Info($"Accepted");
}
}
else
{
resultsDB = new SatyamResultsTableAccess();
resultsDB.UpdateStatusByID(result.ID, ResultStatus.rejected);
resultsDB.close();
WorkerStatisticsManagement.UpdateWorkerStatistics(workerID, taskTemplateType, false, doneTime);
if (logging)
{
log.Info($"Rejected");
}
}
}
//go through all the inconclusive results and check with aggregated table, if aggreagted, compare and see if it is acceptable and then
//accept/reject
public static void AcceptRejectResults()
{
//get all inconclusive results and group them by taskIDs
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
List <SatyamResultsTableEntry> results = resultsDB.getAllEntriesByStatus(ResultStatus.inconclusive);
if (results.Count == 0)
{
resultsDB.close();
return;
}
Dictionary <int, List <SatyamResultsTableEntry> > resultsByTaskID = new Dictionary <int, List <SatyamResultsTableEntry> >();
List <string> pendingGUID = new List <string>();
foreach (SatyamResultsTableEntry result in results)
{
if (!resultsByTaskID.ContainsKey(result.SatyamTaskTableEntryID))
{
resultsByTaskID.Add(result.SatyamTaskTableEntryID, new List <SatyamResultsTableEntry>());
}
resultsByTaskID[result.SatyamTaskTableEntryID].Add(result);
if (!pendingGUID.Contains(result.JobGUID))
{
pendingGUID.Add(result.JobGUID);
}
}
//now check against the aggregated result to see if the result is aceptable
Dictionary <string, Dictionary <int, SatyamAggregatedResultsTableEntry> > aggEntriesPerTaskPerGUID = SatyamAggregatedResultManagement.getAggregatedEntriesPerTaskByGuidList(pendingGUID);
foreach (KeyValuePair <int, List <SatyamResultsTableEntry> > entry in resultsByTaskID)
{
if (entry.Value.Count == 0)
{
continue;
}
int taskEntryID = entry.Key;
string taskGUID = entry.Value[0].JobGUID;
if (aggEntriesPerTaskPerGUID.ContainsKey(taskGUID) && aggEntriesPerTaskPerGUID[taskGUID].ContainsKey(taskEntryID))
{
// this task has been aggregated
foreach (SatyamResultsTableEntry result in resultsByTaskID[taskEntryID]) //now got through each task to see if they satify pass criterion
{
SatyamResult res = JSonUtils.ConvertJSonToObject <SatyamResult>(result.ResultString);
string taskTemplateType = result.JobTemplateType;
string workerID = res.amazonInfo.WorkerID;
DateTime doneTime = res.TaskEndTime;
if (AcceptanceCriterionChecker.IsAcceptable(aggEntriesPerTaskPerGUID[taskGUID][taskEntryID], result))
{
resultsDB.UpdateStatusByID(result.ID, ResultStatus.accepted);
WorkerStatisticsManagement.UpdateWorkerStatistics(workerID, taskTemplateType, true, doneTime);
}
else
{
resultsDB.UpdateStatusByID(result.ID, ResultStatus.rejected);
WorkerStatisticsManagement.UpdateWorkerStatistics(workerID, taskTemplateType, false, doneTime);
}
}
}
}
resultsDB.close();
}
