private async Task GetTask(TaskRunnerInformationBase info, CancellationToken ct)
{
Log?.LogVerbose(GetType(), nameof(RunTask), $"{info.LogPrefix} (CorrelationGroup [{_correlationGroup}]) - Starting task");
DateTime taskStartTime = DateTimeProvider.UtcNow;
long exceptionCount = 0;
while (!ct.IsCancellationRequested)
{
try
{
taskStartTime = DateTimeProvider.UtcNow;
await info.Function().ConfigureAwait(false);
Log?.LogVerbose(GetType(), nameof(RunTask), $"{info.LogPrefix} (CorrelationGroup [{_correlationGroup}]) - Task ran to completion");
return;
}
catch (Exception ex) when(!IsKillerException(ex, ct))
{
var waitTime = Settings.GetWaitTime(exceptionCount);
//If below the MaxUnhandledExceptions count, we simply log a warning.
//Otherwise, if over the max, we attempt to handle via logging an error and calling Exceptionhandler.
if (!ShouldRespondToException(taskStartTime, GetRepeatedTaskFailureTime(ex, info, ct), ref exceptionCount, ct))
{
Log?.LogWarning(GetType(), nameof(RunTask), $"{info.LogPrefix} (CorrelationGroup [{_correlationGroup}]) - Restarting task in [{waitTime.TotalSeconds}s] after [{exceptionCount}] exceptions due to unhandled exception: [{ex}]");
}
else
{
//Log an error, adding exception data to aid in debugging.
Log?.LogError(GetType(), nameof(RunTask), ex, $"{info.LogPrefix} (CorrelationGroup [{_correlationGroup}]) - Restarting task in [{waitTime.TotalSeconds}s] after [{exceptionCount}] exceptions due to unhandled exception.");
//If an exceptionHandler was provided for the task, call it.
if (info.ExceptionHandler != null)
{
try
{
await info.ExceptionHandler(ex, info).ConfigureAwait(false);
}
catch (Exception iex) when(!IsKillerException(ex, ct))
{
Log?.LogWarning(GetType(), nameof(RunTask), $"{info.LogPrefix} (CorrelationGroup [{_correlationGroup}]) - Unhandled exception in final exception handler: [{iex}]");
}
}
}
//Increment the exception count regardless of scenario, and wait for an appropriate period
exceptionCount++;
await Task.Delay(waitTime, ct).ConfigureAwait(false);
}
}
}
/// <summary>
/// Gets the minimum amount of time that a Task should be able to run before encoutering an unhandled exception
/// If an exception is encountered by a Task within this time period after it is restarted, we will start to back
/// off and spend more time in betwen Task restarts.
/// </summary>
/// <param name="ex"></param>
/// <param name="info"></param>
/// <param name="ct"></param>
/// <returns></returns>
private TimeSpan GetRepeatedTaskFailureTime(Exception ex, TaskRunnerInformationBase info, CancellationToken ct)
{
if (info.RepeatedTaskFailureTimeHandler != null)
{
try
{
return(info.RepeatedTaskFailureTimeHandler(ex, Settings.DefaultRepeatedTaskFailureTime));
}
catch (Exception iex) when(!IsKillerException(ex, ct))
{
// Don't want this to throw out, it will go immediately to the final exception. Return the default TimeSpan.
Log?.LogWarning(GetType(), nameof(GetRepeatedTaskFailureTime), $"{info.LogPrefix} (CorrelationGroup [{_correlationGroup}]) - Unhandled exception: [{iex}]");
}
}
return(Settings.DefaultRepeatedTaskFailureTime);
}
private Task RunTask(TaskRunnerInformationBase info, CancellationToken ct)
{
var task = Task.Run(async() =>
{
await GetTask(info, ct).ConfigureAwait(false);
});
task.ContinueWith(t =>
{
// Tasks should only fault on Partition Killer exceptions
// when this happens, cancel all tasks and throw any exceptions
Cancel();
},
ct,
TaskContinuationOptions.ExecuteSynchronously | TaskContinuationOptions.OnlyOnFaulted,
TaskScheduler.Current
);
return(task);
}
