public static void Run([TimerTrigger("0 */5 * * * *")] TimerInfo myTimer, TraceWriter log) { DateTime start = DateTime.Now; bool logging = false; if (logging) { log.Info($"DisposalDispatch executed at: {DateTime.Now}"); } SatyamAmazonHITTableAccess hitDB = new SatyamAmazonHITTableAccess(); // first try and remove all the expired entries List <SatyamAmazonHITTableAccessEntry> expiredHitEntries = hitDB.getAllEntriesByStatus(HitStatus.expired); hitDB.close(); /// dispatch into the queue SatyamDispatchStorageAccountAccess satyamQueue = new SatyamDispatchStorageAccountAccess(); foreach (SatyamAmazonHITTableAccessEntry expiredEntry in expiredHitEntries) { if (logging) { log.Info($"Dispatching Disposal for {expiredEntry.HITID}"); } string queueName = "hit-disposal"; string m = JSonUtils.ConvertObjectToJSon(expiredEntry); satyamQueue.Enqueue(queueName, m); if ((DateTime.Now - start).TotalSeconds > 280) { return; } } hitDB = new SatyamAmazonHITTableAccess(); List <SatyamAmazonHITTableAccessEntry> hitEntriesPending = hitDB.getAllEntriesByStatus(HitStatus.pending); List <SatyamAmazonHITTableAccessEntry> hitEntriesSubmitted = hitDB.getAllEntriesByStatus(HitStatus.submitted); List <SatyamAmazonHITTableAccessEntry> hitEntriesAccepted = hitDB.getAllEntriesByStatus(HitStatus.accepted); List <SatyamAmazonHITTableAccessEntry> hitEntriesRejected = hitDB.getAllEntriesByStatus(HitStatus.rejected); List <SatyamAmazonHITTableAccessEntry> hitEntriesTaken = hitDB.getAllEntriesByStatus(HitStatus.taken); //hitDB.close(); List <SatyamAmazonHITTableAccessEntry> hitEntries = new List <SatyamAmazonHITTableAccessEntry>(); hitEntries.AddRange(hitEntriesPending); hitEntries.AddRange(hitEntriesSubmitted); hitEntries.AddRange(hitEntriesAccepted); hitEntries.AddRange(hitEntriesRejected); hitEntries.AddRange(hitEntriesTaken); if (hitEntries.Count == 0) { return; } //Group them by jobguids since each job can have many tasks Dictionary <string, List <SatyamAmazonHITTableAccessEntry> > HITSByGUID = new Dictionary <string, List <SatyamAmazonHITTableAccessEntry> >(); foreach (SatyamAmazonHITTableAccessEntry hitEntry in hitEntries) { if (hitEntry.EntryExpired()) { //hitDB = new SatyamAmazonHITTableAccess(); hitDB.UpdateStatus(hitEntry.ID, HitStatus.expired); //hitDB.close(); } string guid = hitEntry.JobGUID; if (!HITSByGUID.ContainsKey(hitEntry.JobGUID)) { HITSByGUID.Add(hitEntry.JobGUID, new List <SatyamAmazonHITTableAccessEntry>()); } HITSByGUID[guid].Add(hitEntry); if ((DateTime.Now - start).TotalSeconds > 280) { break; } } hitDB.close(); //now check if this guid has any entries in the task table List <string> guids = HITSByGUID.Keys.ToList(); foreach (string guid in guids) { SatyamTaskTableAccess taskDB = new SatyamTaskTableAccess(); int noTasks = taskDB.getTasksCountByGUID(guid); taskDB.close(); if (noTasks == 0)//remove these additional HITS from the system { List <SatyamAmazonHITTableAccessEntry> hitEntriesToBeRemoved = HITSByGUID[guid]; foreach (SatyamAmazonHITTableAccessEntry hitEntryToBeRemoved in hitEntriesToBeRemoved) { if (logging) { log.Info($"Dispatching Disposal for {hitEntryToBeRemoved.HITID}"); } string queueName = "hit-disposal"; string m = JSonUtils.ConvertObjectToJSon(hitEntryToBeRemoved); satyamQueue.Enqueue(queueName, m); if ((DateTime.Now - start).TotalSeconds > 280) { break; } } } if ((DateTime.Now - start).TotalSeconds > 280) { break; } } if (logging) { log.Info($"DisposalDispatch finished at: {DateTime.Now}"); } }
public static void Run([TimerTrigger("0 */5 * * * *")] TimerInfo myTimer, TraceWriter log) { DateTime start = DateTime.Now; bool logging = false; if (logging) { log.Info($"Payment Dispatch executed at: {DateTime.Now}"); } SatyamDispatchStorageAccountAccess satyamQueue = new SatyamDispatchStorageAccountAccess(); //get IDs of all accepted and rejected results related to MTurk Dictionary <string, string> AssignmentIDToHITIDMap = new Dictionary <string, string>(); List <SatyamResultsTableEntry> acceptedEntries = new List <SatyamResultsTableEntry>(); foreach (string taskTemplate in TaskConstants.MTurkTaskTemplates) { SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); List <SatyamResultsTableEntry> entries = resultsDB.getAllEntriesByJobtemplateTypeAndStatus(taskTemplate, ResultStatus.accepted); resultsDB.close(); acceptedEntries.AddRange(entries); } List <SatyamResultsTableEntry> rejectedEntries = new List <SatyamResultsTableEntry>(); foreach (string taskTemplate in TaskConstants.MTurkTaskTemplates) { SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); List <SatyamResultsTableEntry> entries = resultsDB.getAllEntriesByJobtemplateTypeAndStatus(taskTemplate, ResultStatus.rejected); resultsDB.close(); rejectedEntries.AddRange(entries); } if (acceptedEntries.Count == 0 && rejectedEntries.Count == 0) //nothing to do { return; } // a single assignment may have multiple accepted results Dictionary <string, List <SatyamResultsTableEntry> > acceptedResultsByAssignmentID = new Dictionary <string, List <SatyamResultsTableEntry> >(); foreach (SatyamResultsTableEntry entry in acceptedEntries) { SatyamResult sr = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); string AssignmentID = sr.amazonInfo.AssignmentID; if (!AssignmentIDToHITIDMap.ContainsKey(AssignmentID)) { string HITID = sr.amazonInfo.HITID; AssignmentIDToHITIDMap.Add(AssignmentID, HITID); } if (!acceptedResultsByAssignmentID.ContainsKey(AssignmentID)) { acceptedResultsByAssignmentID.Add(AssignmentID, new List <SatyamResultsTableEntry>()); } acceptedResultsByAssignmentID[AssignmentID].Add(entry); } List <string> assignmentIDList = acceptedResultsByAssignmentID.Keys.ToList(); // a single assignment may have multiple rejected results Dictionary <string, List <SatyamResultsTableEntry> > rejectedResultsByAssignmentID = new Dictionary <string, List <SatyamResultsTableEntry> >(); foreach (SatyamResultsTableEntry entry in rejectedEntries) { SatyamResult sr = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); string AssignmentID = sr.amazonInfo.AssignmentID; if (!AssignmentIDToHITIDMap.ContainsKey(AssignmentID)) { string HITID = sr.amazonInfo.HITID; AssignmentIDToHITIDMap.Add(AssignmentID, HITID); } if (!rejectedResultsByAssignmentID.ContainsKey(AssignmentID)) { rejectedResultsByAssignmentID.Add(AssignmentID, new List <SatyamResultsTableEntry>()); if (!assignmentIDList.Contains(AssignmentID)) { assignmentIDList.Add(AssignmentID); } } rejectedResultsByAssignmentID[AssignmentID].Add(entry); } List <string> rejectedHITs = new List <string>(); List <string> acceptedHITs = new List <string>(); //for each assignment ID we check if it should be paid or not and then pay or reject them. //pay all accepted entries foreach (string assignmentID in assignmentIDList) { int noAccepted = 0; SatyamResult r = null; SatyamResultsTableEntry entry = null; if (acceptedResultsByAssignmentID.ContainsKey(assignmentID)) { noAccepted = acceptedResultsByAssignmentID[assignmentID].Count; entry = acceptedResultsByAssignmentID[assignmentID][0]; r = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); } int noRejected = 0; if (rejectedResultsByAssignmentID.ContainsKey(assignmentID)) { noRejected = rejectedResultsByAssignmentID[assignmentID].Count; if (r == null) { entry = rejectedResultsByAssignmentID[assignmentID][0]; r = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); } } double ratio = (double)noAccepted / ((double)noAccepted + (double)noRejected); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(r.TaskParametersString); string AmazonAccessKeyID = task.jobEntry.amazonHITInformation.AmazonAccessKeyID; string AmazonSecretAccessKeyID = task.jobEntry.amazonHITInformation.AmazonSecretAccessKeyID; string m = AmazonAccessKeyID + "_" + AmazonSecretAccessKeyID + "_" + assignmentID; SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); if (ratio >= AmazonMTurkPayments.assignement_acceptance_threshold) //this is acceptable { if (logging) { log.Info($"Dispatching Payment for {assignmentID}"); } string queueName = "payment"; satyamQueue.Enqueue(queueName, m); if (acceptedResultsByAssignmentID.ContainsKey(assignmentID)) { foreach (SatyamResultsTableEntry result in acceptedResultsByAssignmentID[assignmentID]) { resultsDB.UpdateStatusByID(result.ID, ResultStatus.accepted_Paid); } } if (rejectedResultsByAssignmentID.ContainsKey(assignmentID)) { foreach (SatyamResultsTableEntry result in rejectedResultsByAssignmentID[assignmentID]) { resultsDB.UpdateStatusByID(result.ID, ResultStatus.rejected_Paid); } } if (AssignmentIDToHITIDMap.ContainsKey(assignmentID)) { if (!acceptedHITs.Contains(AssignmentIDToHITIDMap[assignmentID])) { acceptedHITs.Add(AssignmentIDToHITIDMap[assignmentID]); SatyamAmazonHITTableAccess hitDB = new SatyamAmazonHITTableAccess(); hitDB.UpdateStatusByHITID(AssignmentIDToHITIDMap[assignmentID], HitStatus.accepted); hitDB.close(); } } } else { if (logging) { log.Info($"Dispatching NoPayment for {assignmentID}"); } string queueName = "nopayment"; satyamQueue.Enqueue(queueName, m); if (acceptedResultsByAssignmentID.ContainsKey(assignmentID)) { foreach (SatyamResultsTableEntry result in acceptedResultsByAssignmentID[assignmentID]) { resultsDB.UpdateStatusByID(result.ID, ResultStatus.accepted_NotPaid); } } if (rejectedResultsByAssignmentID.ContainsKey(assignmentID)) { foreach (SatyamResultsTableEntry result in rejectedResultsByAssignmentID[assignmentID]) { resultsDB.UpdateStatusByID(result.ID, ResultStatus.rejected_NotPaid); } } if (AssignmentIDToHITIDMap.ContainsKey(assignmentID)) { if (!rejectedHITs.Contains(AssignmentIDToHITIDMap[assignmentID])) { rejectedHITs.Add(AssignmentIDToHITIDMap[assignmentID]); SatyamAmazonHITTableAccess hitDB = new SatyamAmazonHITTableAccess(); hitDB.UpdateStatusByHITID(AssignmentIDToHITIDMap[assignmentID], HitStatus.rejected); hitDB.close(); } } } if ((DateTime.Now - start).TotalSeconds > 280) { break; } } }
public static void Run([TimerTrigger("0 */5 * * * *")] TimerInfo myTimer, TraceWriter log) { DateTime start = DateTime.Now; bool logging = false; if (logging) { log.Info($"Judge Result Dispatch executed at: {DateTime.Now}"); } //get all inconclusive results and group them by taskIDs SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); List <SatyamResultsTableEntry> results = resultsDB.getAllEntriesByStatus(ResultStatus.inconclusive); resultsDB.close(); if (results.Count == 0) { return; } Dictionary <int, List <SatyamResultsTableEntry> > resultsByTaskID = new Dictionary <int, List <SatyamResultsTableEntry> >(); List <string> pendingGUID = new List <string>(); foreach (SatyamResultsTableEntry result in results) { if (!resultsByTaskID.ContainsKey(result.SatyamTaskTableEntryID)) { resultsByTaskID.Add(result.SatyamTaskTableEntryID, new List <SatyamResultsTableEntry>()); } resultsByTaskID[result.SatyamTaskTableEntryID].Add(result); if (!pendingGUID.Contains(result.JobGUID)) { pendingGUID.Add(result.JobGUID); } } //now check against the aggregated result to see if the result is aceptable SatyamDispatchStorageAccountAccess satyamQueue = new SatyamDispatchStorageAccountAccess(); Dictionary <string, Dictionary <int, SatyamAggregatedResultsTableEntry> > aggEntriesPerTaskPerGUID = SatyamAggregatedResultManagement.getAggregatedEntriesPerTaskByGuidList(pendingGUID); foreach (KeyValuePair <int, List <SatyamResultsTableEntry> > entry in resultsByTaskID) { if (entry.Value.Count == 0) { continue; } int taskEntryID = entry.Key; string taskGUID = entry.Value[0].JobGUID; if (aggEntriesPerTaskPerGUID.ContainsKey(taskGUID) && aggEntriesPerTaskPerGUID[taskGUID].ContainsKey(taskEntryID)) { // this task has been aggregated foreach (SatyamResultsTableEntry result in resultsByTaskID[taskEntryID]) //now got through each task to see if they satify pass criterion { if (logging) { log.Info($"Dispatching Judgement for {taskEntryID}"); } string queueName = "judge-result"; string m = taskGUID + "_" + taskEntryID + "_" + result.ID; satyamQueue.Enqueue(queueName, m); if ((DateTime.Now - start).TotalSeconds > 280) { break; } } } if ((DateTime.Now - start).TotalSeconds > 280) { break; } } if (logging) { log.Info($"Judge Result Dispatch finished at: {DateTime.Now}"); } }
public static void Run([TimerTrigger("0 */5 * * * *")] TimerInfo myTimer, TraceWriter log) { DateTime start = DateTime.Now; bool logging = false; if (logging) { log.Info($"Aggregation Dispatch executed at: {DateTime.Now}"); } //first get all the results that are not aggregated SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); List <SatyamResultsTableEntry> results = resultsDB.getEntriesByStatus(ResultStatus.inconclusive); resultsDB.close(); if (results.Count == 0) //there is nothing to do { return; } //first group all the results by the guids, and then by tasks since wach task may have multiple entries // organize by guid, so when polling, each guid will get a chance. SortedDictionary <string, SortedDictionary <int, List <SatyamResultsTableEntry> > > ResultsByGUID = new SortedDictionary <string, SortedDictionary <int, List <SatyamResultsTableEntry> > >(); List <string> pendingGUID = new List <string>(); foreach (SatyamResultsTableEntry entry in results) { string guid = entry.JobGUID; if (!ResultsByGUID.ContainsKey(guid)) { ResultsByGUID.Add(guid, new SortedDictionary <int, List <SatyamResultsTableEntry> >()); } int taskID = entry.SatyamTaskTableEntryID; if (!ResultsByGUID[guid].ContainsKey(taskID)) { ResultsByGUID[guid].Add(taskID, new List <SatyamResultsTableEntry>()); } ResultsByGUID[guid][taskID].Add(entry); if (!pendingGUID.Contains(entry.JobGUID)) { pendingGUID.Add(entry.JobGUID); } } List <string> guidList = ResultsByGUID.Keys.ToList(); if (logging) { log.Info($"Aggregation Dispatch: Results Collected at: {DateTime.Now}"); } //check if guid is still pending SatyamJobSubmissionsTableAccess jobDB = new SatyamJobSubmissionsTableAccess(); List <string> completedGUIDs = jobDB.getAllJobGUIDSByStatus(JobStatus.completed); jobDB.close(); //check if this taskID has already been aggregated and exists int he aggregationDB Dictionary <string, Dictionary <int, SatyamAggregatedResultsTableEntry> > ExistingAggEntriesPerTaskPerGUID = SatyamAggregatedResultManagement.getAggregatedEntriesPerTaskByGuidList(pendingGUID); //Console.WriteLine("Aggregating {0} Tasks", taskIDList.Count); List <SatyamAggregatedResultsTableEntry> aggEntries = new List <SatyamAggregatedResultsTableEntry>(); SatyamDispatchStorageAccountAccess satyamQueue = new SatyamDispatchStorageAccountAccess(); int i = -1; int count = 0; while (true) { i++; bool Done = true; for (int j = 0; j < guidList.Count; j++) { string guid = guidList[j]; List <int> taskIDList = ResultsByGUID[guid].Keys.ToList(); if (taskIDList.Count <= i) { continue; } Done = false; int taskId = taskIDList[i]; if (completedGUIDs.Contains(guid)) { // Hit Completed, mark results outdated. resultsDB = new SatyamResultsTableAccess(); resultsDB.UpdateStatusByTaskID(taskId, ResultStatus.outdated); resultsDB.close(); continue; } //SatyamAggregatedResultsTableAccess aggDB = new SatyamAggregatedResultsTableAccess(); //int LatestResultsAggregated = aggDB.getLatestNoResultsAggregatedByTaskID(taskId); //aggDB.close(); if (ExistingAggEntriesPerTaskPerGUID.ContainsKey(guid) && ExistingAggEntriesPerTaskPerGUID[guid].ContainsKey(taskId)) { continue; //int MinResults = TaskConstants.getMinResultsByTemplate(ExistingAggEntriesPerTaskPerGUID[guid][taskId].JobTemplateType); //if (LatestResultsAggregated >= MinResults) //{ // continue; //} // already aggregated to MinResult request, but leftover results will be judged. So do nothing } //if it does not exist only then aggregate if (logging) { log.Info($"{(DateTime.Now - start).TotalSeconds} Dispatching aggregation for guid {guid} task {taskId}"); } string queueName = "aggregation"; string m = guid + "_" + taskId; satyamQueue.Enqueue(queueName, m); count++; } if (Done) { break; } // emergency break if ((DateTime.Now - start).TotalSeconds > 280) { break; } } if (logging) { log.Info($"Aggregation Dispatch finished at: {DateTime.Now}, dispatched {count} aggregations"); } }