public static void TestManualAcceptReject() { //string path = @"C:\research\listturker.txt"; string path = @"C:\research\topay.txt"; string guid = "dff5415b-b278-43db-8e32-4f654ca4b3a1"; SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); List <SatyamResultsTableEntry> results = resultsDB.getEntriesByGUID(guid); foreach (string line in File.ReadLines(path)) { string[] cs = line.Split('-'); string id = cs[cs.Length - 1].Split('.')[0]; //Console.WriteLine(id); resultsDB.UpdateStatusByID(Convert.ToInt32(id), ResultStatus.accepted); } }
public static void Run([QueueTrigger("judge-result")] string myQueueItem, TraceWriter log) { bool logging = false; string[] fields = myQueueItem.Split('_'); string guid = fields[0]; int taskID = Convert.ToInt32(fields[1]); string resultID = fields[2]; if (logging) { log.Info($"Judge Result: {myQueueItem}"); } SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); List <SatyamResultsTableEntry> results = resultsDB.getEntriesID(resultID); resultsDB.close(); if (results.Count == 0) { return; } SatyamAggregatedResultsTableAccess aggDB = new SatyamAggregatedResultsTableAccess(); SatyamAggregatedResultsTableEntry aggResult = aggDB.getEntryByTaskID(taskID); aggDB.close(); if (aggResult == null) { return; } SatyamResultsTableEntry result = results[0]; SatyamResult res = JSonUtils.ConvertJSonToObject <SatyamResult>(result.ResultString); string taskTemplateType = result.JobTemplateType; string workerID = res.amazonInfo.WorkerID; DateTime doneTime = res.TaskEndTime; if (AcceptanceCriterionChecker.IsAcceptable(aggResult, result)) { resultsDB = new SatyamResultsTableAccess(); resultsDB.UpdateStatusByID(result.ID, ResultStatus.accepted); resultsDB.close(); WorkerStatisticsManagement.UpdateWorkerStatistics(workerID, taskTemplateType, true, doneTime); if (logging) { log.Info($"Accepted"); } } else { resultsDB = new SatyamResultsTableAccess(); resultsDB.UpdateStatusByID(result.ID, ResultStatus.rejected); resultsDB.close(); WorkerStatisticsManagement.UpdateWorkerStatistics(workerID, taskTemplateType, false, doneTime); if (logging) { log.Info($"Rejected"); } } }
public static bool SafeDeleteHit(SatyamAmazonHITTableAccessEntry hitEntryToBeRemoved) { // first expire AmazonMTurkHIT hit = new AmazonMTurkHIT(); hit.setAccount(hitEntryToBeRemoved.AmazonAccessKeyID, hitEntryToBeRemoved.AmazonSecretAccessKeyID, false); hit.expireHIT(hitEntryToBeRemoved.HITID); SatyamAmazonHITTableAccess hitDB = new SatyamAmazonHITTableAccess(); hitDB.UpdateStatus(hitEntryToBeRemoved.ID, HitStatus.expired); hitDB.close(); // then try dispose if (!hit.DeleteHIT(hitEntryToBeRemoved.HITID)) { // can't dispose due to invalid status: reviewable but payment not cleared, or disposed. // because payment not cleared // find all results of this hitID and mark for rejection if (!resultCache.ContainsKey(hitEntryToBeRemoved.JobGUID)) { List <SatyamResultsTableEntry> ret = new List <SatyamResultsTableEntry>(); SatyamResultsTableAccess resDB = new SatyamResultsTableAccess(); List <SatyamResultsTableEntry> tempres = resDB.getEntriesByGUID(hitEntryToBeRemoved.JobGUID); resDB.close(); resultCache.Add(hitEntryToBeRemoved.JobGUID, tempres); } List <SatyamResultsTableEntry> results = resultCache[hitEntryToBeRemoved.JobGUID]; List <SatyamResultsTableEntry> res = new List <SatyamResultsTableEntry>(); foreach (SatyamResultsTableEntry r in results) { SatyamResult rr = JSonUtils.ConvertJSonToObject <SatyamResult>(r.ResultString); if (rr.amazonInfo.HITID == hitEntryToBeRemoved.HITID) { res.Add(r); } } if (res.Count == 0) { // the hit can't be disposed because there are pending assignments, yet we don't have any on records, // so delete it. AWS will automatically handle it after 120 days of inactivity. hitDB = new SatyamAmazonHITTableAccess(); hitDB.DeleteEntry(hitEntryToBeRemoved.ID); hitDB.close(); Console.WriteLine("Delete Success\n"); return(true); } //bool HITDone = true; foreach (SatyamResultsTableEntry r in res) { //SatyamResult rr = JSonUtils.ConvertJSonToObject<SatyamResult>(r.ResultString); //string assignmentID = rr.amazonInfo.AssignmentID; //hit.RejectAssignment(assignmentID, "Sorry! Your work was not within acceptable parameters!"); if (r.Status == ResultStatus.inconclusive || r.Status == ResultStatus.outdated) { // reject all for now... TODO: check if there is an aggregated results. //Console.WriteLine("Rejecting result ID {0}", r.ID); //SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); //resultsDB.UpdateStatusByID(r.ID, ResultStatus.rejected); //resultsDB.close(); //HITDone = false; } else { Console.WriteLine("Result ID {0} is of status {1}", r.ID, r.Status); if (r.Status == ResultStatus.accepted || r.Status == ResultStatus.rejected) { // need to be handled by payment function //HITDone = false; } // just in case last payment was not made successfully, // mark it to be paid again.... SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); if (ResultStatus.acceptedStatusList.Contains(r.Status)) { Console.WriteLine("Marking as Accepted so Next Cycle will Pay"); resultsDB.UpdateStatusByID(r.ID, ResultStatus.accepted); } if (ResultStatus.rejectedStatusList.Contains(r.Status)) { Console.WriteLine("Marking as Rejected so Next Cycle will Reject"); resultsDB.UpdateStatusByID(r.ID, ResultStatus.rejected); } resultsDB.close(); } } // Commented out cuz the case where the inconsistency status happen is rare, // so don't delete the hit too aggressively //if (HITDone) //{ // hitDB = new SatyamAmazonHITTableAccess(); // hitDB.DeleteEntry(hitEntryToBeRemoved.ID); // hitDB.close(); // Console.WriteLine("Delete Success\n"); // return true; //} Console.WriteLine("Delete Failure\n"); return(false); } else { hitDB = new SatyamAmazonHITTableAccess(); hitDB.DeleteEntry(hitEntryToBeRemoved.ID); hitDB.close(); Console.WriteLine("Delete Success\n"); return(true); } }
//go through all unpaid results //check if aggregated results has an entry for this //check if it satisfies acceptance criterion //pay if passes else fail! public static void MakePayments() { //get IDs of all accepted and rejected results related to MTurk SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); Dictionary <string, string> AssignmentIDToHITIDMap = new Dictionary <string, string>(); List <SatyamResultsTableEntry> acceptedEntries = new List <SatyamResultsTableEntry>(); foreach (string taskTemplate in TaskConstants.MTurkTaskTemplates) { List <SatyamResultsTableEntry> entries = resultsDB.getAllEntriesByJobtemplateTypeAndStatus(taskTemplate, ResultStatus.accepted); acceptedEntries.AddRange(entries); } List <SatyamResultsTableEntry> rejectedEntries = new List <SatyamResultsTableEntry>(); foreach (string taskTemplate in TaskConstants.MTurkTaskTemplates) { List <SatyamResultsTableEntry> entries = resultsDB.getAllEntriesByJobtemplateTypeAndStatus(taskTemplate, ResultStatus.rejected); rejectedEntries.AddRange(entries); } resultsDB.close(); if (acceptedEntries.Count == 0 && rejectedEntries.Count == 0) //nothing to do { return; } // a single assignment may have multiple accepted results Dictionary <string, List <SatyamResultsTableEntry> > acceptedResultsByAssignmentID = new Dictionary <string, List <SatyamResultsTableEntry> >(); foreach (SatyamResultsTableEntry entry in acceptedEntries) { SatyamResult sr = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); string AssignmentID = sr.amazonInfo.AssignmentID; if (!AssignmentIDToHITIDMap.ContainsKey(AssignmentID)) { string HITID = sr.amazonInfo.HITID; AssignmentIDToHITIDMap.Add(AssignmentID, HITID); } if (!acceptedResultsByAssignmentID.ContainsKey(AssignmentID)) { acceptedResultsByAssignmentID.Add(AssignmentID, new List <SatyamResultsTableEntry>()); } acceptedResultsByAssignmentID[AssignmentID].Add(entry); } List <string> assignmentIDList = acceptedResultsByAssignmentID.Keys.ToList(); // a single assignment may have multiple rejected results Dictionary <string, List <SatyamResultsTableEntry> > rejectedResultsByAssignmentID = new Dictionary <string, List <SatyamResultsTableEntry> >(); foreach (SatyamResultsTableEntry entry in rejectedEntries) { SatyamResult sr = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); string AssignmentID = sr.amazonInfo.AssignmentID; if (!AssignmentIDToHITIDMap.ContainsKey(AssignmentID)) { string HITID = sr.amazonInfo.HITID; AssignmentIDToHITIDMap.Add(AssignmentID, HITID); } if (!rejectedResultsByAssignmentID.ContainsKey(AssignmentID)) { rejectedResultsByAssignmentID.Add(AssignmentID, new List <SatyamResultsTableEntry>()); if (!assignmentIDList.Contains(AssignmentID)) { assignmentIDList.Add(AssignmentID); } } rejectedResultsByAssignmentID[AssignmentID].Add(entry); } List <string> rejectedHITs = new List <string>(); List <string> acceptedHITs = new List <string>(); //for each assignment ID we check if it should be paid or not and then pay or reject them. //pay all accepted entries AmazonMTurkHIT hit = new AmazonMTurkHIT(); foreach (string assignmentID in assignmentIDList) { int noAccepted = 0; SatyamResult r = null; SatyamResultsTableEntry entry = null; if (acceptedResultsByAssignmentID.ContainsKey(assignmentID)) { noAccepted = acceptedResultsByAssignmentID[assignmentID].Count; entry = acceptedResultsByAssignmentID[assignmentID][0]; r = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); } int noRejected = 0; if (rejectedResultsByAssignmentID.ContainsKey(assignmentID)) { noRejected = rejectedResultsByAssignmentID[assignmentID].Count; if (r == null) { entry = rejectedResultsByAssignmentID[assignmentID][0]; r = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); } } double ratio = (double)noAccepted / ((double)noAccepted + (double)noRejected); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(r.TaskParametersString); string AmazonAccessKeyID = task.jobEntry.amazonHITInformation.AmazonAccessKeyID; string AmazonSecretAccessKeyID = task.jobEntry.amazonHITInformation.AmazonSecretAccessKeyID; hit.setAccount(AmazonAccessKeyID, AmazonSecretAccessKeyID, false); resultsDB = new SatyamResultsTableAccess(); if (ratio >= assignement_acceptance_threshold) //this is acceptable { hit.ApproveAssignment(assignmentID, "Great Job! Your work was within acceptable parameters!"); if (acceptedResultsByAssignmentID.ContainsKey(assignmentID)) { foreach (SatyamResultsTableEntry result in acceptedResultsByAssignmentID[assignmentID]) { resultsDB.UpdateStatusByID(result.ID, ResultStatus.accepted_Paid); } } if (rejectedResultsByAssignmentID.ContainsKey(assignmentID)) { foreach (SatyamResultsTableEntry result in rejectedResultsByAssignmentID[assignmentID]) { resultsDB.UpdateStatusByID(result.ID, ResultStatus.rejected_Paid); } } if (AssignmentIDToHITIDMap.ContainsKey(assignmentID)) { if (!acceptedHITs.Contains(AssignmentIDToHITIDMap[assignmentID])) { acceptedHITs.Add(AssignmentIDToHITIDMap[assignmentID]); } } } else { hit.RejectAssignment(assignmentID, "Sorry! Your work was not within acceptable parameters!"); if (acceptedResultsByAssignmentID.ContainsKey(assignmentID)) { foreach (SatyamResultsTableEntry result in acceptedResultsByAssignmentID[assignmentID]) { resultsDB.UpdateStatusByID(result.ID, ResultStatus.accepted_NotPaid); } } if (rejectedResultsByAssignmentID.ContainsKey(assignmentID)) { foreach (SatyamResultsTableEntry result in rejectedResultsByAssignmentID[assignmentID]) { resultsDB.UpdateStatusByID(result.ID, ResultStatus.rejected_NotPaid); } } if (AssignmentIDToHITIDMap.ContainsKey(assignmentID)) { if (!rejectedHITs.Contains(AssignmentIDToHITIDMap[assignmentID])) { rejectedHITs.Add(AssignmentIDToHITIDMap[assignmentID]); } } } resultsDB.close(); } //update the status of all finished HITs in the hittable SatyamAmazonHITTableAccess hitDB = new SatyamAmazonHITTableAccess(); foreach (string HITID in acceptedHITs) { hitDB.UpdateStatusByHITID(HITID, HitStatus.accepted); } foreach (string HITID in rejectedHITs) { hitDB.UpdateStatusByHITID(HITID, HitStatus.rejected); } hitDB.close(); }
//go through all the inconclusive results and check with aggregated table, if aggreagted, compare and see if it is acceptable and then //accept/reject public static void AcceptRejectResults() { //get all inconclusive results and group them by taskIDs SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); List <SatyamResultsTableEntry> results = resultsDB.getAllEntriesByStatus(ResultStatus.inconclusive); if (results.Count == 0) { resultsDB.close(); return; } Dictionary <int, List <SatyamResultsTableEntry> > resultsByTaskID = new Dictionary <int, List <SatyamResultsTableEntry> >(); List <string> pendingGUID = new List <string>(); foreach (SatyamResultsTableEntry result in results) { if (!resultsByTaskID.ContainsKey(result.SatyamTaskTableEntryID)) { resultsByTaskID.Add(result.SatyamTaskTableEntryID, new List <SatyamResultsTableEntry>()); } resultsByTaskID[result.SatyamTaskTableEntryID].Add(result); if (!pendingGUID.Contains(result.JobGUID)) { pendingGUID.Add(result.JobGUID); } } //now check against the aggregated result to see if the result is aceptable Dictionary <string, Dictionary <int, SatyamAggregatedResultsTableEntry> > aggEntriesPerTaskPerGUID = SatyamAggregatedResultManagement.getAggregatedEntriesPerTaskByGuidList(pendingGUID); foreach (KeyValuePair <int, List <SatyamResultsTableEntry> > entry in resultsByTaskID) { if (entry.Value.Count == 0) { continue; } int taskEntryID = entry.Key; string taskGUID = entry.Value[0].JobGUID; if (aggEntriesPerTaskPerGUID.ContainsKey(taskGUID) && aggEntriesPerTaskPerGUID[taskGUID].ContainsKey(taskEntryID)) { // this task has been aggregated foreach (SatyamResultsTableEntry result in resultsByTaskID[taskEntryID]) //now got through each task to see if they satify pass criterion { SatyamResult res = JSonUtils.ConvertJSonToObject <SatyamResult>(result.ResultString); string taskTemplateType = result.JobTemplateType; string workerID = res.amazonInfo.WorkerID; DateTime doneTime = res.TaskEndTime; if (AcceptanceCriterionChecker.IsAcceptable(aggEntriesPerTaskPerGUID[taskGUID][taskEntryID], result)) { resultsDB.UpdateStatusByID(result.ID, ResultStatus.accepted); WorkerStatisticsManagement.UpdateWorkerStatistics(workerID, taskTemplateType, true, doneTime); } else { resultsDB.UpdateStatusByID(result.ID, ResultStatus.rejected); WorkerStatisticsManagement.UpdateWorkerStatistics(workerID, taskTemplateType, false, doneTime); } } } } resultsDB.close(); }
public static void Run([TimerTrigger("0 */5 * * * *")] TimerInfo myTimer, TraceWriter log) { DateTime start = DateTime.Now; bool logging = false; if (logging) { log.Info($"Payment Dispatch executed at: {DateTime.Now}"); } SatyamDispatchStorageAccountAccess satyamQueue = new SatyamDispatchStorageAccountAccess(); //get IDs of all accepted and rejected results related to MTurk Dictionary <string, string> AssignmentIDToHITIDMap = new Dictionary <string, string>(); List <SatyamResultsTableEntry> acceptedEntries = new List <SatyamResultsTableEntry>(); foreach (string taskTemplate in TaskConstants.MTurkTaskTemplates) { SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); List <SatyamResultsTableEntry> entries = resultsDB.getAllEntriesByJobtemplateTypeAndStatus(taskTemplate, ResultStatus.accepted); resultsDB.close(); acceptedEntries.AddRange(entries); } List <SatyamResultsTableEntry> rejectedEntries = new List <SatyamResultsTableEntry>(); foreach (string taskTemplate in TaskConstants.MTurkTaskTemplates) { SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); List <SatyamResultsTableEntry> entries = resultsDB.getAllEntriesByJobtemplateTypeAndStatus(taskTemplate, ResultStatus.rejected); resultsDB.close(); rejectedEntries.AddRange(entries); } if (acceptedEntries.Count == 0 && rejectedEntries.Count == 0) //nothing to do { return; } // a single assignment may have multiple accepted results Dictionary <string, List <SatyamResultsTableEntry> > acceptedResultsByAssignmentID = new Dictionary <string, List <SatyamResultsTableEntry> >(); foreach (SatyamResultsTableEntry entry in acceptedEntries) { SatyamResult sr = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); string AssignmentID = sr.amazonInfo.AssignmentID; if (!AssignmentIDToHITIDMap.ContainsKey(AssignmentID)) { string HITID = sr.amazonInfo.HITID; AssignmentIDToHITIDMap.Add(AssignmentID, HITID); } if (!acceptedResultsByAssignmentID.ContainsKey(AssignmentID)) { acceptedResultsByAssignmentID.Add(AssignmentID, new List <SatyamResultsTableEntry>()); } acceptedResultsByAssignmentID[AssignmentID].Add(entry); } List <string> assignmentIDList = acceptedResultsByAssignmentID.Keys.ToList(); // a single assignment may have multiple rejected results Dictionary <string, List <SatyamResultsTableEntry> > rejectedResultsByAssignmentID = new Dictionary <string, List <SatyamResultsTableEntry> >(); foreach (SatyamResultsTableEntry entry in rejectedEntries) { SatyamResult sr = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); string AssignmentID = sr.amazonInfo.AssignmentID; if (!AssignmentIDToHITIDMap.ContainsKey(AssignmentID)) { string HITID = sr.amazonInfo.HITID; AssignmentIDToHITIDMap.Add(AssignmentID, HITID); } if (!rejectedResultsByAssignmentID.ContainsKey(AssignmentID)) { rejectedResultsByAssignmentID.Add(AssignmentID, new List <SatyamResultsTableEntry>()); if (!assignmentIDList.Contains(AssignmentID)) { assignmentIDList.Add(AssignmentID); } } rejectedResultsByAssignmentID[AssignmentID].Add(entry); } List <string> rejectedHITs = new List <string>(); List <string> acceptedHITs = new List <string>(); //for each assignment ID we check if it should be paid or not and then pay or reject them. //pay all accepted entries foreach (string assignmentID in assignmentIDList) { int noAccepted = 0; SatyamResult r = null; SatyamResultsTableEntry entry = null; if (acceptedResultsByAssignmentID.ContainsKey(assignmentID)) { noAccepted = acceptedResultsByAssignmentID[assignmentID].Count; entry = acceptedResultsByAssignmentID[assignmentID][0]; r = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); } int noRejected = 0; if (rejectedResultsByAssignmentID.ContainsKey(assignmentID)) { noRejected = rejectedResultsByAssignmentID[assignmentID].Count; if (r == null) { entry = rejectedResultsByAssignmentID[assignmentID][0]; r = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); } } double ratio = (double)noAccepted / ((double)noAccepted + (double)noRejected); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(r.TaskParametersString); string AmazonAccessKeyID = task.jobEntry.amazonHITInformation.AmazonAccessKeyID; string AmazonSecretAccessKeyID = task.jobEntry.amazonHITInformation.AmazonSecretAccessKeyID; string m = AmazonAccessKeyID + "_" + AmazonSecretAccessKeyID + "_" + assignmentID; SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); if (ratio >= AmazonMTurkPayments.assignement_acceptance_threshold) //this is acceptable { if (logging) { log.Info($"Dispatching Payment for {assignmentID}"); } string queueName = "payment"; satyamQueue.Enqueue(queueName, m); if (acceptedResultsByAssignmentID.ContainsKey(assignmentID)) { foreach (SatyamResultsTableEntry result in acceptedResultsByAssignmentID[assignmentID]) { resultsDB.UpdateStatusByID(result.ID, ResultStatus.accepted_Paid); } } if (rejectedResultsByAssignmentID.ContainsKey(assignmentID)) { foreach (SatyamResultsTableEntry result in rejectedResultsByAssignmentID[assignmentID]) { resultsDB.UpdateStatusByID(result.ID, ResultStatus.rejected_Paid); } } if (AssignmentIDToHITIDMap.ContainsKey(assignmentID)) { if (!acceptedHITs.Contains(AssignmentIDToHITIDMap[assignmentID])) { acceptedHITs.Add(AssignmentIDToHITIDMap[assignmentID]); SatyamAmazonHITTableAccess hitDB = new SatyamAmazonHITTableAccess(); hitDB.UpdateStatusByHITID(AssignmentIDToHITIDMap[assignmentID], HitStatus.accepted); hitDB.close(); } } } else { if (logging) { log.Info($"Dispatching NoPayment for {assignmentID}"); } string queueName = "nopayment"; satyamQueue.Enqueue(queueName, m); if (acceptedResultsByAssignmentID.ContainsKey(assignmentID)) { foreach (SatyamResultsTableEntry result in acceptedResultsByAssignmentID[assignmentID]) { resultsDB.UpdateStatusByID(result.ID, ResultStatus.accepted_NotPaid); } } if (rejectedResultsByAssignmentID.ContainsKey(assignmentID)) { foreach (SatyamResultsTableEntry result in rejectedResultsByAssignmentID[assignmentID]) { resultsDB.UpdateStatusByID(result.ID, ResultStatus.rejected_NotPaid); } } if (AssignmentIDToHITIDMap.ContainsKey(assignmentID)) { if (!rejectedHITs.Contains(AssignmentIDToHITIDMap[assignmentID])) { rejectedHITs.Add(AssignmentIDToHITIDMap[assignmentID]); SatyamAmazonHITTableAccess hitDB = new SatyamAmazonHITTableAccess(); hitDB.UpdateStatusByHITID(AssignmentIDToHITIDMap[assignmentID], HitStatus.rejected); hitDB.close(); } } } if ((DateTime.Now - start).TotalSeconds > 280) { break; } } }