private static async Task <T> ExecuteWithRetriesHelper <T>(
Func <int, Task <T> > function,
int callCounter,
int maxNumSuccessTries,
int maxNumErrorTries,
TimeSpan maxExecutionTime,
DateTime startExecutionTime,
Func <T, int, bool> retryValueFilter = null,
Func <Exception, int, bool> retryExceptionFilter = null,
IBackoffProvider onSuccessBackOff = null,
IBackoffProvider onErrorBackOff = null)
{
if (maxExecutionTime != Constants.INFINITE_TIMESPAN && maxExecutionTime != default(TimeSpan))
{
DateTime now = DateTime.UtcNow;
if (now - startExecutionTime > maxExecutionTime)
{
Exception timeoutException = new TimeoutException(String.Format("ExecuteWithRetries has exceeded its max execution time of {0}. Now is {1}, started at {2}, passed {3}",
maxExecutionTime, TraceLogger.PrintDate(now), TraceLogger.PrintDate(startExecutionTime), now - startExecutionTime));
throw timeoutException;
}
}
T result = default(T);
int counter = callCounter;
Exception exception = null;
try
{
callCounter++;
result = await function(counter);
bool retry = false;
if (callCounter < maxNumSuccessTries || maxNumSuccessTries == INFINITE_RETRIES) // -1 for infinite retries
{
if (retryValueFilter != null)
{
retry = retryValueFilter(result, counter);
}
}
if (retry)
{
if (onSuccessBackOff == null)
{
return(await ExecuteWithRetriesHelper(function, callCounter, maxNumSuccessTries, maxNumErrorTries, maxExecutionTime, startExecutionTime, retryValueFilter, retryExceptionFilter, onSuccessBackOff, onErrorBackOff));
}
else
{
TimeSpan delay = onSuccessBackOff.Next();
await Task.Delay(delay);
return(await ExecuteWithRetriesHelper(function, callCounter, maxNumSuccessTries, maxNumErrorTries, maxExecutionTime, startExecutionTime, retryValueFilter, retryExceptionFilter, onSuccessBackOff, onErrorBackOff));
}
}
return(result);
}
catch (Exception exc)
{
exception = exc;
}
if (exception != null)
{
bool retry = false;
if (callCounter < maxNumErrorTries || maxNumErrorTries == INFINITE_RETRIES)
{
if (retryExceptionFilter != null)
{
retry = retryExceptionFilter(exception, counter);
}
}
if (retry)
{
if (onErrorBackOff == null)
{
return(await ExecuteWithRetriesHelper(function, callCounter, maxNumSuccessTries, maxNumErrorTries, maxExecutionTime, startExecutionTime, retryValueFilter, retryExceptionFilter, onSuccessBackOff, onErrorBackOff));
}
else
{
TimeSpan delay = onErrorBackOff.Next();
await Task.Delay(delay);
return(await ExecuteWithRetriesHelper(function, callCounter, maxNumSuccessTries, maxNumErrorTries, maxExecutionTime, startExecutionTime, retryValueFilter, retryExceptionFilter, onSuccessBackOff, onErrorBackOff));
}
}
throw exception;
}
return(result); // this return value is just for the compiler to supress "not all control paths return a value".
}
private static async Task <T> ExecuteWithRetriesHelper <T>(
Func <int, Task <T> > function,
int callCounter,
int maxNumSuccessTries,
int maxNumErrorTries,
TimeSpan maxExecutionTime,
DateTime startExecutionTime,
Func <T, int, bool> retryValueFilter = null,
Func <Exception, int, bool> retryExceptionFilter = null,
IBackoffProvider onSuccessBackOff = null,
IBackoffProvider onErrorBackOff = null)
{
T result = default(T);
ExceptionDispatchInfo lastExceptionInfo = null;
bool retry;
do
{
retry = false;
if (maxExecutionTime != Constants.INFINITE_TIMESPAN && maxExecutionTime != default(TimeSpan))
{
DateTime now = DateTime.UtcNow;
if (now - startExecutionTime > maxExecutionTime)
{
if (lastExceptionInfo == null)
{
throw new TimeoutException(
$"ExecuteWithRetries has exceeded its max execution time of {maxExecutionTime}. Now is {LogFormatter.PrintDate(now)}, started at {LogFormatter.PrintDate(startExecutionTime)}, passed {now - startExecutionTime}");
}
lastExceptionInfo.Throw();
}
}
int counter = callCounter;
try
{
callCounter++;
result = await function(counter);
lastExceptionInfo = null;
if (callCounter < maxNumSuccessTries || maxNumSuccessTries == INFINITE_RETRIES) // -1 for infinite retries
{
if (retryValueFilter != null)
{
retry = retryValueFilter(result, counter);
}
}
if (retry)
{
TimeSpan?delay = onSuccessBackOff?.Next(counter);
if (delay.HasValue)
{
await Task.Delay(delay.Value);
}
}
}
catch (Exception exc)
{
retry = false;
if (callCounter < maxNumErrorTries || maxNumErrorTries == INFINITE_RETRIES)
{
if (retryExceptionFilter != null)
{
retry = retryExceptionFilter(exc, counter);
}
}
if (!retry)
{
throw;
}
lastExceptionInfo = ExceptionDispatchInfo.Capture(exc);
TimeSpan?delay = onErrorBackOff?.Next(counter);
if (delay.HasValue)
{
await Task.Delay(delay.Value);
}
}
} while (retry);
return(result);
}