private void WriteOperations()
{
try
{
StringBuilder builder = new StringBuilder();
if (Running > 0)
{
builder.AppendLine($"--------------------\nRunning Operations: {Running}\n--------------------");
lock (RunningOperations)
{
RunningOperations.GroupBy(x => x.Operation.Name).Select(x => new { Name = x.Key, Count = x.Count() }).ToList().ForEach(x => builder.AppendLine($"{x.Name}: {x.Count}"));
}
}
if (Successes > 0)
{
builder.AppendLine($"--------------------\nSuccessful Operations: {Successes}\n--------------------");
SuccessfulOperations.GroupBy(x => x.Operation.Name).Select(x => new { Name = x.Key, Count = x.Count() }).ToList().ForEach(x => builder.AppendLine($"{x.Name}: {x.Count}"));
}
if (Failures > 0)
{
builder.AppendLine($"--------------------\nFailed Operations: {Failures}\n--------------------");
FailedOperations.GroupBy(x => x.Operation.Name).Select(x => new { Name = x.Key, Count = x.Count() }).ToList().ForEach(x => builder.AppendLine($"{x.Name}: {x.Count}"));
}
builder.AppendLine();
WriteToBaseLog(builder.ToString());
}
catch
{
// ignored
}
}
private async void RunOperation(OperationContext operationContext)
{
CancellationTokenSource source = new CancellationTokenSource();
cancellationTokens.Add(operationContext.Operation.Name + operationContext.RunDate.ToShortDateString(), source);
DateTime startTime = DateTime.Now;
StringBuilder logBuilder = new StringBuilder();
#if Debug
void LogMessage(String s)
{
logBuilder.AppendLine($"[{DateTime.Now:MM/dd/yyyy h:mm:ss.fff tt}] {s}");
Console.WriteLine(s);
}
#else
void LogMessage(String s) => logBuilder.AppendLine(string.Format("[{0:MM/dd/yyyy h:mm:ss.fff tt}] {1}", DateTime.Now, s));
#endif
using (IDatabaseRepository <IHarvesterDataContext> harvester = RepositoryFactory.CreateHarvesterRepository(repositories["Harvester"]))
{
LogMessage($"Connected to database '{harvester.Name}' ({harvester.ConnectionString})");
Operation operation = harvester.DataContext.Operations.FirstOrDefault(x => x.Name == operationContext.Operation.Name);
if (operation == null)
{
operation = new Operation {
Name = operationContext.Operation.Name
};
harvester.DataContext.Operations.InsertOnSubmit(operation);
harvester.DataContext.SubmitChanges();
operationContext.Operation.OperationID = operation.ID;
}
else
{
operationContext.Operation.OperationID = operation.ID;
if (operation.OperationRecords.Any(x => x.RunDate.Date == operationContext.RunDate.Date))
{
skippedOperations++;
WriteQueueManagerChanges($"Operation {operationContext.Operation.Name.PadRight(33)} Has already Run");
harvester.DataContext.SubmitChanges();
return;
}
}
}
LogMessage($"Starting operation '{operationContext.Operation.Name}' with run date of {operationContext.RunDate:MM/dd/yyyy h:mm:ss.fff tt}");
TrackOperationsByDate(operationContext.Operation.OperationID);
Task task = Task.Run(async() =>
{
source.Token.ThrowIfCancellationRequested();
await IsTooManyOperationsRunning(operationContext, LogMessage);
await ReachedMaximumRunsPerDay(operationContext, LogMessage);
await IsTooManyConcurrentlyRunning(operationContext, LogMessage);
lock (RunningOperations)
{
RunningOperations.Add(operationContext);
}
WriteCurrentStatus();
startTime = DateTime.Now;
source.Token.ThrowIfCancellationRequested();
WriteQueueManagerChanges($"operation {operationContext.Operation.Name.PadRight(33)} Began Executing");
operationContext.Operation.Execute(operationContext.RunDate, LogMessage, source.Token);
});
Exception exception = null;
try
{
LogMessage($"Awaiting operation {operationContext.Operation.Name}");
await task;
}
catch (Exception ex)
{
exception = ex;
LogMessage(ex.ToString());
}
finally
{
TotalOperationsRun++;
lock (RunningOperations)
{
RunningOperations.Remove(operationContext);
}
cancellationTokens.Remove(operationContext.Operation.Name + operationContext.RunDate.ToShortDateString());
DateTime endDate = DateTime.Now;
char statusLetter;
switch (task.Status)
{
case TaskStatus.Canceled:
WriteQueueManagerChanges($"operation {operationContext.Operation.Name.PadRight(33)} canceled");
LogMessage("The operation was cancelled.");
CanceledOperations.Add(operationContext);
statusLetter = 'C';
break;
case TaskStatus.Faulted:
WriteQueueManagerChanges($"operation {operationContext.Operation.Name.PadRight(33)} Faulted");
LogMessage("The operation encountered an error.");
FailedOperations.Add(operationContext);
statusLetter = 'F';
break;
case TaskStatus.RanToCompletion:
WriteQueueManagerChanges($"operation {operationContext.Operation.Name.PadRight(33)} Ran to Completion");
LogMessage("The operation completed successfully.");
SuccessfulOperations.Add(operationContext);
statusLetter = 'S';
using (IDatabaseRepository <IHarvesterDataContext> harvester = RepositoryFactory.CreateHarvesterRepository(repositories["Harvester"]))
{
OperationRecord operationRecord = new OperationRecord
{
OperationID = operationContext.Operation.OperationID,
RunDate = operationContext.RunDate,
ExecutedDate = DateTime.Now
};
LogMessage($"Updated Harvester with Operation Record {JsonConvert.SerializeObject(operationRecord)}");
harvester.DataContext.OperationRecords.InsertOnSubmit(operationRecord);
harvester.DataContext.SubmitChanges();
}
break;
default:
WriteQueueManagerChanges($"operation {operationContext.Operation.Name} has an unexpected task status of: {task.Status}");
throw new NotImplementedException($"operation {operationContext.Operation.Name} has an unexpected task status of: {task.Status}");
}
RetriedOperations.Remove(operationContext);
LogMessage("The operation took " + endDate.Subtract(startTime));
string filename = $"{baseDirectory}Logs\\" +
$"({statusLetter}) " +
$"{operationContext.Operation.Name} " +
$"({operationContext.RunDate:yyyy-MM-dd}) " +
$"{(operationContext.CurrentRetry == 0 ? "" : "(" + operationContext.CurrentRetry + ")")} " +
$"{Guid.NewGuid()}.txt";
HarvesterService.WriteToFile(filename, logBuilder.ToString());
logBuilder.Clear();
WriteCurrentStatus();
WriteOperations();
switch (task.Status)
{
#pragma warning disable CS4014 // Because this call is not awaited, execution of the current method continues before the call is completed
case TaskStatus.Faulted:
Task.Run(async() =>
{
if (exception?.GetType() == typeof(RepositoryIOException))
{
await Task.Delay(TimeSpan.FromSeconds(((RepositoryIOException)exception).RetryWaitTime), source.Token);
}
else
{
await Task.Delay(TimeSpan.FromDays(1), source.Token);
}
RetriedOperations.Add(operationContext);
RunOperation(operationContext.GetRetry());
});
break;
#pragma warning restore CS4014 // Because this call is not awaited, execution of the current method continues before the call is completed
case TaskStatus.RanToCompletion:
case TaskStatus.Canceled:
break;
default:
WriteQueueManagerChanges($"operation {operationContext.Operation.Name} has an unexpected task status of: {task.Status}");
throw new NotImplementedException($"operation {operationContext.Operation.Name} has an unexpected task status of: {task.Status}");
}
}
}
