private void WriteOperations() { try { StringBuilder builder = new StringBuilder(); if (Running > 0) { builder.AppendLine($"--------------------\nRunning Operations: {Running}\n--------------------"); lock (RunningOperations) { RunningOperations.GroupBy(x => x.Operation.Name).Select(x => new { Name = x.Key, Count = x.Count() }).ToList().ForEach(x => builder.AppendLine($"{x.Name}: {x.Count}")); } } if (Successes > 0) { builder.AppendLine($"--------------------\nSuccessful Operations: {Successes}\n--------------------"); SuccessfulOperations.GroupBy(x => x.Operation.Name).Select(x => new { Name = x.Key, Count = x.Count() }).ToList().ForEach(x => builder.AppendLine($"{x.Name}: {x.Count}")); } if (Failures > 0) { builder.AppendLine($"--------------------\nFailed Operations: {Failures}\n--------------------"); FailedOperations.GroupBy(x => x.Operation.Name).Select(x => new { Name = x.Key, Count = x.Count() }).ToList().ForEach(x => builder.AppendLine($"{x.Name}: {x.Count}")); } builder.AppendLine(); WriteToBaseLog(builder.ToString()); } catch { // ignored } }
private async Task IsTooManyConcurrentlyRunning(OperationContext operationContext, Action <string> logMessage) { if (operationContext.MaximumConcurrentlyRunning != 0) { int minutesDelayed = 0; bool maximumOperationsRunning; lock (RunningOperations) { maximumOperationsRunning = RunningOperations.Count(x => x.Operation.Name == operationContext.Operation.Name) >= operationContext.MaximumConcurrentlyRunning; } while (maximumOperationsRunning) { minutesDelayed += minutesToDelayForConcurrency; requestsDelayedForConcurrency++; await Task.Delay(TimeSpan.FromMinutes(minutesToDelayForConcurrency)); requestsDelayedForConcurrency--; lock (RunningOperations) { maximumOperationsRunning = RunningOperations.Count(x => x.Operation.Name == operationContext.Operation.Name) >= operationContext.MaximumConcurrentlyRunning; } } if (minutesDelayed != 0) { logMessage($"Delayed operation '{operationContext.Operation.Name}' for {minutesDelayed} minutes until it could be run without any other operations of the same type running"); } } }
private void AddOperation(OperationContext operationContext) { lock (queuedOperations) if (RunningOperations != null) { lock (RunningOperations) { if (queuedOperations.Contains(operationContext)) { throw new ArgumentOutOfRangeException(nameof(operationContext), "This operation has already been queued to run."); } if (RunningOperations.Contains(operationContext)) { throw new ArgumentOutOfRangeException(nameof(operationContext), "This operation is already running."); } } } queuedOperations.Enqueue(operationContext); }
private async void RunOperation(OperationContext operationContext) { CancellationTokenSource source = new CancellationTokenSource(); cancellationTokens.Add(operationContext.Operation.Name + operationContext.RunDate.ToShortDateString(), source); DateTime startTime = DateTime.Now; StringBuilder logBuilder = new StringBuilder(); #if Debug void LogMessage(String s) { logBuilder.AppendLine($"[{DateTime.Now:MM/dd/yyyy h:mm:ss.fff tt}] {s}"); Console.WriteLine(s); } #else void LogMessage(String s) => logBuilder.AppendLine(string.Format("[{0:MM/dd/yyyy h:mm:ss.fff tt}] {1}", DateTime.Now, s)); #endif using (IDatabaseRepository <IHarvesterDataContext> harvester = RepositoryFactory.CreateHarvesterRepository(repositories["Harvester"])) { LogMessage($"Connected to database '{harvester.Name}' ({harvester.ConnectionString})"); Operation operation = harvester.DataContext.Operations.FirstOrDefault(x => x.Name == operationContext.Operation.Name); if (operation == null) { operation = new Operation { Name = operationContext.Operation.Name }; harvester.DataContext.Operations.InsertOnSubmit(operation); harvester.DataContext.SubmitChanges(); operationContext.Operation.OperationID = operation.ID; } else { operationContext.Operation.OperationID = operation.ID; if (operation.OperationRecords.Any(x => x.RunDate.Date == operationContext.RunDate.Date)) { skippedOperations++; WriteQueueManagerChanges($"Operation {operationContext.Operation.Name.PadRight(33)} Has already Run"); harvester.DataContext.SubmitChanges(); return; } } } LogMessage($"Starting operation '{operationContext.Operation.Name}' with run date of {operationContext.RunDate:MM/dd/yyyy h:mm:ss.fff tt}"); TrackOperationsByDate(operationContext.Operation.OperationID); Task task = Task.Run(async() => { source.Token.ThrowIfCancellationRequested(); await IsTooManyOperationsRunning(operationContext, LogMessage); await ReachedMaximumRunsPerDay(operationContext, LogMessage); await IsTooManyConcurrentlyRunning(operationContext, LogMessage); lock (RunningOperations) { RunningOperations.Add(operationContext); } WriteCurrentStatus(); startTime = DateTime.Now; source.Token.ThrowIfCancellationRequested(); WriteQueueManagerChanges($"operation {operationContext.Operation.Name.PadRight(33)} Began Executing"); operationContext.Operation.Execute(operationContext.RunDate, LogMessage, source.Token); }); Exception exception = null; try { LogMessage($"Awaiting operation {operationContext.Operation.Name}"); await task; } catch (Exception ex) { exception = ex; LogMessage(ex.ToString()); } finally { TotalOperationsRun++; lock (RunningOperations) { RunningOperations.Remove(operationContext); } cancellationTokens.Remove(operationContext.Operation.Name + operationContext.RunDate.ToShortDateString()); DateTime endDate = DateTime.Now; char statusLetter; switch (task.Status) { case TaskStatus.Canceled: WriteQueueManagerChanges($"operation {operationContext.Operation.Name.PadRight(33)} canceled"); LogMessage("The operation was cancelled."); CanceledOperations.Add(operationContext); statusLetter = 'C'; break; case TaskStatus.Faulted: WriteQueueManagerChanges($"operation {operationContext.Operation.Name.PadRight(33)} Faulted"); LogMessage("The operation encountered an error."); FailedOperations.Add(operationContext); statusLetter = 'F'; break; case TaskStatus.RanToCompletion: WriteQueueManagerChanges($"operation {operationContext.Operation.Name.PadRight(33)} Ran to Completion"); LogMessage("The operation completed successfully."); SuccessfulOperations.Add(operationContext); statusLetter = 'S'; using (IDatabaseRepository <IHarvesterDataContext> harvester = RepositoryFactory.CreateHarvesterRepository(repositories["Harvester"])) { OperationRecord operationRecord = new OperationRecord { OperationID = operationContext.Operation.OperationID, RunDate = operationContext.RunDate, ExecutedDate = DateTime.Now }; LogMessage($"Updated Harvester with Operation Record {JsonConvert.SerializeObject(operationRecord)}"); harvester.DataContext.OperationRecords.InsertOnSubmit(operationRecord); harvester.DataContext.SubmitChanges(); } break; default: WriteQueueManagerChanges($"operation {operationContext.Operation.Name} has an unexpected task status of: {task.Status}"); throw new NotImplementedException($"operation {operationContext.Operation.Name} has an unexpected task status of: {task.Status}"); } RetriedOperations.Remove(operationContext); LogMessage("The operation took " + endDate.Subtract(startTime)); string filename = $"{baseDirectory}Logs\\" + $"({statusLetter}) " + $"{operationContext.Operation.Name} " + $"({operationContext.RunDate:yyyy-MM-dd}) " + $"{(operationContext.CurrentRetry == 0 ? "" : "(" + operationContext.CurrentRetry + ")")} " + $"{Guid.NewGuid()}.txt"; HarvesterService.WriteToFile(filename, logBuilder.ToString()); logBuilder.Clear(); WriteCurrentStatus(); WriteOperations(); switch (task.Status) { #pragma warning disable CS4014 // Because this call is not awaited, execution of the current method continues before the call is completed case TaskStatus.Faulted: Task.Run(async() => { if (exception?.GetType() == typeof(RepositoryIOException)) { await Task.Delay(TimeSpan.FromSeconds(((RepositoryIOException)exception).RetryWaitTime), source.Token); } else { await Task.Delay(TimeSpan.FromDays(1), source.Token); } RetriedOperations.Add(operationContext); RunOperation(operationContext.GetRetry()); }); break; #pragma warning restore CS4014 // Because this call is not awaited, execution of the current method continues before the call is completed case TaskStatus.RanToCompletion: case TaskStatus.Canceled: break; default: WriteQueueManagerChanges($"operation {operationContext.Operation.Name} has an unexpected task status of: {task.Status}"); throw new NotImplementedException($"operation {operationContext.Operation.Name} has an unexpected task status of: {task.Status}"); } } }