/// <summary>
/// Executes and, if necessary, retries an operation
/// </summary>
protected async Task <U> RetryAsync <U>(
string operationName,
Func <T, CancellationToken, Task <U> > fn,
CancellationToken cancellationToken,
IEnumerator <TimeSpan> retryIntervalEnumerator = null,
bool reloadFirst = false,
Guid?operationId = null,
TimeSpan?timeout = null)
{
operationId = operationId ?? Guid.NewGuid();
retryIntervalEnumerator = retryIntervalEnumerator ?? m_retryIntervals.GetEnumerator();
timeout = timeout ?? s_defaultOperationTimeout;
try
{
using (CancellationTokenSource timeoutCancellationSource = new CancellationTokenSource())
using (CancellationTokenSource innerCancellationSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, timeoutCancellationSource.Token))
{
var instance = GetCurrentVersionedValue();
if (reloadFirst)
{
var reloaded = Reloader.Reload(instance.Version);
m_logger.Warning("[{2}] Service client reloaded; new instance created: {0}, new client version: {1}", reloaded, Reloader.CurrentVersion, operationId.Value);
}
m_logger.Verbose("[{2}] Invoking '{0}' against instance version {1}", operationName, instance.Version, operationId.Value);
return(await WithTimeoutAsync(fn(instance.Value, innerCancellationSource.Token), timeout.Value, timeoutCancellationSource));
}
}
catch (Exception e) when(m_nonRetryableExceptions.Contains(e.GetType()))
{
// We should not retry exceptions of this type.
throw;
}
catch (Exception e)
{
if (e is TimeoutException)
{
m_logger.Warning("Timeout ({0}sec) happened while waiting {1}.", timeout.Value.TotalSeconds, operationName);
}
if (retryIntervalEnumerator.MoveNext())
{
m_logger.Warning("[{2}] Waiting {1} before retrying on exception: {0}", e.ToString(), retryIntervalEnumerator.Current, operationId.Value);
await Task.Delay(retryIntervalEnumerator.Current);
return(await RetryAsync(operationName, fn, cancellationToken, retryIntervalEnumerator, reloadFirst : true, operationId : operationId));
}
else
{
m_logger.Error("[{1}] Failing because number of retries were exhausted. Final exception: {0};", e.ToString(), operationId.Value);
throw;
}
}
}
private async Task <T> RetryAsync <T>(
string operationName,
Func <IDropServiceClient, CancellationToken, Task <T> > fn,
CancellationToken cancellationToken,
IEnumerator <TimeSpan> retryIntervalEnumerator = null,
bool reloadFirst = false,
Guid?operationId = null,
TimeSpan?timeout = null)
{
operationId = operationId ?? Guid.NewGuid();
retryIntervalEnumerator = retryIntervalEnumerator ?? m_retryIntervals.GetEnumerator();
timeout = timeout ?? s_defaultOperationTimeout;
try
{
using (CancellationTokenSource timeoutCancellationSource = new CancellationTokenSource())
using (CancellationTokenSource innerCancellationSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, timeoutCancellationSource.Token))
{
var instance = GetCurrentVersionedValue();
if (reloadFirst)
{
var reloaded = m_reloader.Reload(instance.Version);
m_logger.Warning("[{2}] Drop service client reloaded; new instance created: {0}, new client version: {1}", reloaded, m_reloader.CurrentVersion, operationId.Value);
}
m_logger.Verbose("[{2}] Invoking '{0}' against IDropServiceClient instance version {1}", operationName, instance.Version, operationId.Value);
return(await WithTimeoutAsync(fn(instance.Value, innerCancellationSource.Token), timeout.Value, timeoutCancellationSource));
}
}
catch (Exception e)
{
if (e is TimeoutException)
{
m_logger.Warning("Timeout ({0}sec) happened while waiting {1}.", timeout.Value.TotalSeconds, operationName);
}
if (retryIntervalEnumerator.MoveNext())
{
m_logger.Warning("[{2}] Exception caught: {0} Waiting {1} before retrying.", e.ToString(), retryIntervalEnumerator.Current, operationId.Value);
await Task.Delay(retryIntervalEnumerator.Current);
return(await RetryAsync(operationName, fn, cancellationToken, retryIntervalEnumerator, reloadFirst : true, operationId : operationId));
}
else
{
m_logger.Warning("[{1}] Exception caught: {0}; failing because number of retries were exhausted", e.ToString(), operationId.Value);
throw;
}
}
}
