public static void Run([TimerTrigger("0 */5 * * * *")] TimerInfo myTimer, TraceWriter log) { DateTime start = DateTime.Now; bool logging = false; if (logging) { log.Info($"Judge Result Dispatch executed at: {DateTime.Now}"); } //get all inconclusive results and group them by taskIDs SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); List <SatyamResultsTableEntry> results = resultsDB.getAllEntriesByStatus(ResultStatus.inconclusive); resultsDB.close(); if (results.Count == 0) { return; } Dictionary <int, List <SatyamResultsTableEntry> > resultsByTaskID = new Dictionary <int, List <SatyamResultsTableEntry> >(); List <string> pendingGUID = new List <string>(); foreach (SatyamResultsTableEntry result in results) { if (!resultsByTaskID.ContainsKey(result.SatyamTaskTableEntryID)) { resultsByTaskID.Add(result.SatyamTaskTableEntryID, new List <SatyamResultsTableEntry>()); } resultsByTaskID[result.SatyamTaskTableEntryID].Add(result); if (!pendingGUID.Contains(result.JobGUID)) { pendingGUID.Add(result.JobGUID); } } //now check against the aggregated result to see if the result is aceptable SatyamDispatchStorageAccountAccess satyamQueue = new SatyamDispatchStorageAccountAccess(); Dictionary <string, Dictionary <int, SatyamAggregatedResultsTableEntry> > aggEntriesPerTaskPerGUID = SatyamAggregatedResultManagement.getAggregatedEntriesPerTaskByGuidList(pendingGUID); foreach (KeyValuePair <int, List <SatyamResultsTableEntry> > entry in resultsByTaskID) { if (entry.Value.Count == 0) { continue; } int taskEntryID = entry.Key; string taskGUID = entry.Value[0].JobGUID; if (aggEntriesPerTaskPerGUID.ContainsKey(taskGUID) && aggEntriesPerTaskPerGUID[taskGUID].ContainsKey(taskEntryID)) { // this task has been aggregated foreach (SatyamResultsTableEntry result in resultsByTaskID[taskEntryID]) //now got through each task to see if they satify pass criterion { if (logging) { log.Info($"Dispatching Judgement for {taskEntryID}"); } string queueName = "judge-result"; string m = taskGUID + "_" + taskEntryID + "_" + result.ID; satyamQueue.Enqueue(queueName, m); if ((DateTime.Now - start).TotalSeconds > 280) { break; } } } if ((DateTime.Now - start).TotalSeconds > 280) { break; } } if (logging) { log.Info($"Judge Result Dispatch finished at: {DateTime.Now}"); } }
public static void Run([TimerTrigger("0 */5 * * * *")] TimerInfo myTimer, TraceWriter log) { DateTime start = DateTime.Now; bool logging = false; if (logging) { log.Info($"Aggregation Dispatch executed at: {DateTime.Now}"); } //first get all the results that are not aggregated SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); List <SatyamResultsTableEntry> results = resultsDB.getEntriesByStatus(ResultStatus.inconclusive); resultsDB.close(); if (results.Count == 0) //there is nothing to do { return; } //first group all the results by the guids, and then by tasks since wach task may have multiple entries // organize by guid, so when polling, each guid will get a chance. SortedDictionary <string, SortedDictionary <int, List <SatyamResultsTableEntry> > > ResultsByGUID = new SortedDictionary <string, SortedDictionary <int, List <SatyamResultsTableEntry> > >(); List <string> pendingGUID = new List <string>(); foreach (SatyamResultsTableEntry entry in results) { string guid = entry.JobGUID; if (!ResultsByGUID.ContainsKey(guid)) { ResultsByGUID.Add(guid, new SortedDictionary <int, List <SatyamResultsTableEntry> >()); } int taskID = entry.SatyamTaskTableEntryID; if (!ResultsByGUID[guid].ContainsKey(taskID)) { ResultsByGUID[guid].Add(taskID, new List <SatyamResultsTableEntry>()); } ResultsByGUID[guid][taskID].Add(entry); if (!pendingGUID.Contains(entry.JobGUID)) { pendingGUID.Add(entry.JobGUID); } } List <string> guidList = ResultsByGUID.Keys.ToList(); if (logging) { log.Info($"Aggregation Dispatch: Results Collected at: {DateTime.Now}"); } //check if guid is still pending SatyamJobSubmissionsTableAccess jobDB = new SatyamJobSubmissionsTableAccess(); List <string> completedGUIDs = jobDB.getAllJobGUIDSByStatus(JobStatus.completed); jobDB.close(); //check if this taskID has already been aggregated and exists int he aggregationDB Dictionary <string, Dictionary <int, SatyamAggregatedResultsTableEntry> > ExistingAggEntriesPerTaskPerGUID = SatyamAggregatedResultManagement.getAggregatedEntriesPerTaskByGuidList(pendingGUID); //Console.WriteLine("Aggregating {0} Tasks", taskIDList.Count); List <SatyamAggregatedResultsTableEntry> aggEntries = new List <SatyamAggregatedResultsTableEntry>(); SatyamDispatchStorageAccountAccess satyamQueue = new SatyamDispatchStorageAccountAccess(); int i = -1; int count = 0; while (true) { i++; bool Done = true; for (int j = 0; j < guidList.Count; j++) { string guid = guidList[j]; List <int> taskIDList = ResultsByGUID[guid].Keys.ToList(); if (taskIDList.Count <= i) { continue; } Done = false; int taskId = taskIDList[i]; if (completedGUIDs.Contains(guid)) { // Hit Completed, mark results outdated. resultsDB = new SatyamResultsTableAccess(); resultsDB.UpdateStatusByTaskID(taskId, ResultStatus.outdated); resultsDB.close(); continue; } //SatyamAggregatedResultsTableAccess aggDB = new SatyamAggregatedResultsTableAccess(); //int LatestResultsAggregated = aggDB.getLatestNoResultsAggregatedByTaskID(taskId); //aggDB.close(); if (ExistingAggEntriesPerTaskPerGUID.ContainsKey(guid) && ExistingAggEntriesPerTaskPerGUID[guid].ContainsKey(taskId)) { continue; //int MinResults = TaskConstants.getMinResultsByTemplate(ExistingAggEntriesPerTaskPerGUID[guid][taskId].JobTemplateType); //if (LatestResultsAggregated >= MinResults) //{ // continue; //} // already aggregated to MinResult request, but leftover results will be judged. So do nothing } //if it does not exist only then aggregate if (logging) { log.Info($"{(DateTime.Now - start).TotalSeconds} Dispatching aggregation for guid {guid} task {taskId}"); } string queueName = "aggregation"; string m = guid + "_" + taskId; satyamQueue.Enqueue(queueName, m); count++; } if (Done) { break; } // emergency break if ((DateTime.Now - start).TotalSeconds > 280) { break; } } if (logging) { log.Info($"Aggregation Dispatch finished at: {DateTime.Now}, dispatched {count} aggregations"); } }