/// <summary>
/// Execute a given function a number of times, based on retry configuration parameters.
/// </summary>
public static Task <T> ExecuteWithRetries <T>(
Func <int, Task <T> > function,
int maxNumErrorTries,
Func <Exception, int, bool> retryExceptionFilter,
TimeSpan maxExecutionTime,
IBackoffProvider onErrorBackOff)
{
return(ExecuteWithRetries <T>(
function,
0,
maxNumErrorTries,
null,
retryExceptionFilter,
maxExecutionTime,
null,
onErrorBackOff));
}
/// <summary>
/// Execute a given function a number of times, based on retry configuration parameters.
/// </summary>
/// <param name="action">
/// The action to be executed.
/// </param>
/// <param name="maxNumErrorTries">
/// The maximum number of retries.
/// </param>
/// <param name="retryExceptionFilter">
/// The retry exception filter.
/// </param>
/// <param name="maxExecutionTime">
/// The maximum execution time.
/// </param>
/// <param name="onErrorBackOff">
/// The backoff provider.
/// </param>
/// <returns>
/// A <see cref="Task"/> representing the operation.
/// </returns>
public static Task ExecuteWithRetries(
Func <int, Task> action,
int maxNumErrorTries,
Func <Exception, int, bool> retryExceptionFilter,
TimeSpan maxExecutionTime,
IBackoffProvider onErrorBackOff)
{
Func <int, Task <bool> > function = async(int i) => { await action(i); return(true); };
return(ExecuteWithRetriesHelper <bool>(
function,
0,
maxNumErrorTries,
maxExecutionTime,
DateTime.UtcNow,
null,
retryExceptionFilter,
null,
onErrorBackOff));
}
/// <summary>
/// Execute a given <paramref name="function"/> a number of times, based on retry configuration parameters.
/// </summary>
/// <typeparam name="T">
/// The underlying return type of <paramref name="function"/>.
/// </typeparam>
/// <param name="function">
/// Function to execute
/// </param>
/// <param name="maxNumSuccessTries">
/// Maximal number of successful execution attempts. <see cref="ExecuteWithRetries"/> will try to re-execute the given <paramref name="function"/> again if directed so by <paramref name="retryValueFilter"/> .
/// Set to <c>-1</c> for unlimited number of success retries, until <paramref name="retryValueFilter"/> is satisfied. Set to <c>0</c> for only one success attempt, which will cause <paramref name="retryValueFilter"/> to be
/// ignored and the given <paramref name="function"/> executed only once until first success.
/// </param>
/// <param name="maxNumErrorTries">
/// Maximal number of execution attempts due to errors. Set to -1 for unlimited number of error retries, until <paramref name="retryExceptionFilter"/> is satisfied.
/// </param>
/// <param name="retryValueFilter">
/// Filter <paramref name="function"/> to indicate if successful execution should be retried. Set to <see langword="null"/> to disable successful retries.
/// </param>
/// <param name="retryExceptionFilter">
/// Filter <paramref name="function"/> to indicate if error execution should be retried. Set to <see langword="null"/> to disable error retries.
/// </param>
/// <param name="maxExecutionTime">
/// The maximal execution time of the <see cref="ExecuteWithRetries"/> function.
/// </param>
/// <param name="onSuccessBackOff">
/// The backoff provider object, which determines how much to wait between success retries.
/// </param>
/// <param name="onErrorBackOff">
/// The backoff provider object, which determines how much to wait between error retries
/// </param>
/// <returns>
/// The value returned from the successful invocation of <paramref name="function"/>.
/// </returns>
public static Task <T> ExecuteWithRetries <T>(
Func <int, Task <T> > function,
int maxNumSuccessTries,
int maxNumErrorTries,
Func <T, int, bool> retryValueFilter,
Func <Exception, int, bool> retryExceptionFilter,
TimeSpan maxExecutionTime = default(TimeSpan),
IBackoffProvider onSuccessBackOff = null,
IBackoffProvider onErrorBackOff = null)
{
return(ExecuteWithRetriesHelper <T>(
function,
maxNumSuccessTries,
maxNumErrorTries,
maxExecutionTime,
DateTime.UtcNow,
retryValueFilter,
retryExceptionFilter,
onSuccessBackOff,
onErrorBackOff));
}
private static async Task <T> ExecuteWithRetriesHelper <T>(
Func <int, Task <T> > function,
int callCounter,
int maxNumSuccessTries,
int maxNumErrorTries,
TimeSpan maxExecutionTime,
DateTime startExecutionTime,
Func <T, int, bool> retryValueFilter = null,
Func <Exception, int, bool> retryExceptionFilter = null,
IBackoffProvider onSuccessBackOff = null,
IBackoffProvider onErrorBackOff = null)
{
if (maxExecutionTime != Constants.INFINITE_TIMESPAN && maxExecutionTime != default(TimeSpan))
{
DateTime now = DateTime.UtcNow;
if (now - startExecutionTime > maxExecutionTime)
{
Exception timeoutException = new TimeoutException(String.Format("ExecuteWithRetries has exceeded its max execution time of {0}. Now is {1}, started at {2}, passed {3}",
maxExecutionTime, TraceLogger.PrintDate(now), TraceLogger.PrintDate(startExecutionTime), now - startExecutionTime));
throw timeoutException;
}
}
T result = default(T);
int counter = callCounter;
Exception exception = null;
try
{
callCounter++;
result = await function(counter);
bool retry = false;
if (callCounter < maxNumSuccessTries || maxNumSuccessTries == INFINITE_RETRIES) // -1 for infinite retries
{
if (retryValueFilter != null)
{
retry = retryValueFilter(result, counter);
}
}
if (retry)
{
if (onSuccessBackOff == null)
{
return(await ExecuteWithRetriesHelper(function, callCounter, maxNumSuccessTries, maxNumErrorTries, maxExecutionTime, startExecutionTime, retryValueFilter, retryExceptionFilter, onSuccessBackOff, onErrorBackOff));
}
else
{
TimeSpan delay = onSuccessBackOff.Next();
await Task.Delay(delay);
return(await ExecuteWithRetriesHelper(function, callCounter, maxNumSuccessTries, maxNumErrorTries, maxExecutionTime, startExecutionTime, retryValueFilter, retryExceptionFilter, onSuccessBackOff, onErrorBackOff));
}
}
return(result);
}
catch (Exception exc)
{
exception = exc;
}
if (exception != null)
{
bool retry = false;
if (callCounter < maxNumErrorTries || maxNumErrorTries == INFINITE_RETRIES)
{
if (retryExceptionFilter != null)
{
retry = retryExceptionFilter(exception, counter);
}
}
if (retry)
{
if (onErrorBackOff == null)
{
return(await ExecuteWithRetriesHelper(function, callCounter, maxNumSuccessTries, maxNumErrorTries, maxExecutionTime, startExecutionTime, retryValueFilter, retryExceptionFilter, onSuccessBackOff, onErrorBackOff));
}
else
{
TimeSpan delay = onErrorBackOff.Next();
await Task.Delay(delay);
return(await ExecuteWithRetriesHelper(function, callCounter, maxNumSuccessTries, maxNumErrorTries, maxExecutionTime, startExecutionTime, retryValueFilter, retryExceptionFilter, onSuccessBackOff, onErrorBackOff));
}
}
throw exception;
}
return(result); // this return value is just for the compiler to supress "not all control paths return a value".
}
private static async Task <T> ExecuteWithRetriesHelper <T>(
Func <int, Task <T> > function,
int callCounter,
int maxNumSuccessTries,
int maxNumErrorTries,
TimeSpan maxExecutionTime,
DateTime startExecutionTime,
Func <T, int, bool> retryValueFilter = null,
Func <Exception, int, bool> retryExceptionFilter = null,
IBackoffProvider onSuccessBackOff = null,
IBackoffProvider onErrorBackOff = null)
{
T result = default(T);
ExceptionDispatchInfo lastExceptionInfo = null;
bool retry;
do
{
retry = false;
if (maxExecutionTime != Constants.INFINITE_TIMESPAN && maxExecutionTime != default(TimeSpan))
{
DateTime now = DateTime.UtcNow;
if (now - startExecutionTime > maxExecutionTime)
{
if (lastExceptionInfo == null)
{
throw new TimeoutException(
$"ExecuteWithRetries has exceeded its max execution time of {maxExecutionTime}. Now is {LogFormatter.PrintDate(now)}, started at {LogFormatter.PrintDate(startExecutionTime)}, passed {now - startExecutionTime}");
}
lastExceptionInfo.Throw();
}
}
int counter = callCounter;
try
{
callCounter++;
result = await function(counter);
lastExceptionInfo = null;
if (callCounter < maxNumSuccessTries || maxNumSuccessTries == INFINITE_RETRIES) // -1 for infinite retries
{
if (retryValueFilter != null)
{
retry = retryValueFilter(result, counter);
}
}
if (retry)
{
TimeSpan?delay = onSuccessBackOff?.Next(counter);
if (delay.HasValue)
{
await Task.Delay(delay.Value);
}
}
}
catch (Exception exc)
{
retry = false;
if (callCounter < maxNumErrorTries || maxNumErrorTries == INFINITE_RETRIES)
{
if (retryExceptionFilter != null)
{
retry = retryExceptionFilter(exc, counter);
}
}
if (!retry)
{
throw;
}
lastExceptionInfo = ExceptionDispatchInfo.Capture(exc);
TimeSpan?delay = onErrorBackOff?.Next(counter);
if (delay.HasValue)
{
await Task.Delay(delay.Value);
}
}
} while (retry);
return(result);
}
