static async Task <TResult> Attempt <T, TResult>(RetryContext <T> retryContext, Func <Task <TResult> > retryMethod, CancellationToken cancellationToken)
where T : class
{
cancellationToken.ThrowIfCancellationRequested();
try
{
return(await retryMethod().ConfigureAwait(false));
}
catch (Exception exception)
{
RetryContext <T> nextRetryContext;
if (!retryContext.CanRetry(exception, out nextRetryContext))
{
throw;
}
if (nextRetryContext.Delay.HasValue)
{
await Task.Delay(nextRetryContext.Delay.Value, cancellationToken).ConfigureAwait(false);
}
return(await Attempt(nextRetryContext, retryMethod, cancellationToken).ConfigureAwait(false));
}
}
async Task Attempt(RetryContext <TContext> retryContext, IPipe <TContext> next)
{
await retryContext.PreRetry().ConfigureAwait(false);
try
{
await next.Send(retryContext.Context).ConfigureAwait(false);
}
catch (Exception exception)
{
RetryContext <TContext> payloadRetryContext;
if (retryContext.Context.TryGetPayload(out payloadRetryContext) && !payloadRetryContext.CanRetry(exception, out payloadRetryContext))
{
await retryContext.RetryFaulted(exception).ConfigureAwait(false);
throw;
}
RetryContext <TContext> nextRetryContext;
if (!retryContext.CanRetry(exception, out nextRetryContext))
{
await nextRetryContext.RetryFaulted(exception).ConfigureAwait(false);
retryContext.Context.GetOrAddPayload(() => nextRetryContext);
throw;
}
if (nextRetryContext.Delay.HasValue)
{
await Task.Delay(nextRetryContext.Delay.Value).ConfigureAwait(false);
}
await Attempt(nextRetryContext, next).ConfigureAwait(false);
}
}
static async Task Attempt <T>(RetryContext <ConsumeContext <T> > retryContext, Func <ConsumeContext <T>, Task> retryMethod)
where T : class
{
try
{
await retryMethod(retryContext.Context).ConfigureAwait(false);
}
catch (Exception exception)
{
if (retryContext.Context.TryGetPayload(out RetryContext <ConsumeContext <T> > nextRetryContext))
{
throw;
}
if (!retryContext.CanRetry(exception, out nextRetryContext))
{
retryContext.Context.GetOrAddPayload(() => nextRetryContext);
throw;
}
if (nextRetryContext.Delay.HasValue)
{
await Task.Delay(nextRetryContext.Delay.Value).ConfigureAwait(false);
}
await Attempt(nextRetryContext, retryMethod).ConfigureAwait(false);
}
}
public bool CanRetry(Exception exception, out RetryContext <CommandContext> retryContext)
{
var canRetry = _retryContext.CanRetry(exception, out RetryContext <CommandContext> policyRetryContext);
retryContext = new ConsumeContextRetryContext(policyRetryContext, canRetry ? _context.CreateNext() : _context);
return(canRetry);
}
async Task Run()
{
var stoppingContext = new TransportStoppingContext(Stopping);
RetryPolicyContext <TransportStoppingContext> policyContext = _retryPolicy.CreatePolicyContext(stoppingContext);
try
{
while (!IsStopping)
{
RetryContext <TransportStoppingContext> retryContext = null;
try
{
if (retryContext != null)
{
LogContext.Warning?.Log(retryContext.Exception, "Retrying {Delay}: {Message}", retryContext.Delay,
retryContext.Exception.Message);
if (retryContext.Delay.HasValue)
{
await Task.Delay(retryContext.Delay.Value, Stopping).ConfigureAwait(false);
}
}
if (!IsStopping)
{
await RunTransport().ConfigureAwait(false);
}
}
catch (OperationCanceledException)
{
throw;
}
catch (Exception exception)
{
if (retryContext != null)
{
retryContext = retryContext.CanRetry(exception, out RetryContext <TransportStoppingContext> nextRetryContext)
? nextRetryContext
: null;
}
if (retryContext == null && !policyContext.CanRetry(exception, out retryContext))
{
break;
}
}
}
}
catch (Exception exception)
{
LogContext.Debug?.Log(exception, "ReceiveTransport Run Exception: {InputAddress}", _context.InputAddress);
}
finally
{
policyContext.Dispose();
}
}
public bool CanRetry(Exception exception, out RetryContext <ConsumeContext> retryContext)
{
RetryContext <ConsumeContext> policyRetryContext;
var canRetry = _retryContext.CanRetry(exception, out policyRetryContext);
retryContext = new ConsumeContextRetryContext(policyRetryContext, _context);
return(canRetry);
}
static async Task Attempt <T>(RetryContext <T> retryContext, Func <Task> retryMethod)
where T : class
{
while (retryContext.CancellationToken.IsCancellationRequested == false)
{
if (retryContext.Delay.HasValue)
{
await Task.Delay(retryContext.Delay.Value, retryContext.CancellationToken).ConfigureAwait(false);
}
retryContext.CancellationToken.ThrowIfCancellationRequested();
try
{
await retryMethod().ConfigureAwait(false);
return;
}
catch (Exception exception)
{
if (retryContext.CancellationToken.IsCancellationRequested)
{
var canceledException = exception as OperationCanceledException;
if (canceledException != null && canceledException.CancellationToken == retryContext.CancellationToken)
{
throw;
}
retryContext.CancellationToken.ThrowIfCancellationRequested();
}
RetryContext <T> nextRetryContext;
if (!retryContext.CanRetry(exception, out nextRetryContext))
{
throw;
}
retryContext = nextRetryContext;
}
}
retryContext.CancellationToken.ThrowIfCancellationRequested();
}
static async Task <TResult> Attempt <T, TResult>(T context, RetryContext <T> retryContext, Func <Task <TResult> > retryMethod)
where T : class, PipeContext
{
while (context.CancellationToken.IsCancellationRequested == false)
{
if (retryContext.Delay.HasValue)
{
await Task.Delay(retryContext.Delay.Value, context.CancellationToken).ConfigureAwait(false);
}
context.CancellationToken.ThrowIfCancellationRequested();
try
{
return(await retryMethod().ConfigureAwait(false));
}
catch (Exception exception)
{
if (context.CancellationToken.IsCancellationRequested)
{
var canceledException = exception as OperationCanceledException;
if (canceledException != null && canceledException.CancellationToken == context.CancellationToken)
{
throw;
}
context.CancellationToken.ThrowIfCancellationRequested();
}
RetryContext <T> nextRetryContext;
if (!retryContext.CanRetry(exception, out nextRetryContext))
{
throw;
}
retryContext = nextRetryContext;
}
}
context.CancellationToken.ThrowIfCancellationRequested();
throw new OperationCanceledException("Retry was cancelled");
}
static async Task Attempt <T>(T context, RetryContext <T> retryContext, Func <Task> retryMethod)
where T : class, PipeContext
{
while (context.CancellationToken.IsCancellationRequested == false)
{
if (retryContext.Delay.HasValue)
{
await Task.Delay(retryContext.Delay.Value, context.CancellationToken).ConfigureAwait(false);
}
context.CancellationToken.ThrowIfCancellationRequested();
try
{
await retryMethod().ConfigureAwait(false);
return;
}
catch (Exception exception)
{
if (context.CancellationToken.IsCancellationRequested)
{
if (exception is OperationCanceledException canceledException && canceledException.CancellationToken == context.CancellationToken)
{
throw;
}
context.CancellationToken.ThrowIfCancellationRequested();
}
if (!retryContext.CanRetry(exception, out RetryContext <T> nextRetryContext))
{
throw;
}
retryContext = nextRetryContext;
}
}
context.CancellationToken.ThrowIfCancellationRequested();
}
static async Task Attempt <T>(T context, RetryContext <T> retryContext, Func <Task> retryMethod)
where T : class, PipeContext
{
while (context.CancellationToken.IsCancellationRequested == false)
{
LogContext.Warning?.Log(retryContext.Exception, "Retrying {Delay}: {Message}", retryContext.Delay, retryContext.Exception.Message);
try
{
if (retryContext.Delay.HasValue)
{
await Task.Delay(retryContext.Delay.Value, context.CancellationToken).ConfigureAwait(false);
}
if (!context.CancellationToken.IsCancellationRequested)
{
await retryMethod().ConfigureAwait(false);
}
return;
}
catch (OperationCanceledException exception) when(exception.CancellationToken == context.CancellationToken)
{
retryContext.Exception?.Rethrow();
throw;
}
catch (Exception exception)
{
if (!retryContext.CanRetry(exception, out RetryContext <T> nextRetryContext))
{
throw new OperationCanceledException(context.CancellationToken);
}
retryContext = nextRetryContext;
}
}
throw new OperationCanceledException(context.CancellationToken);
}
async Task Attempt(TContext context, RetryContext <TContext> retryContext, IPipe <TContext> next)
{
while (retryContext.CancellationToken.IsCancellationRequested == false)
{
if (retryContext.Delay.HasValue)
{
await Task.Delay(retryContext.Delay.Value, retryContext.CancellationToken).ConfigureAwait(false);
}
await retryContext.PreRetry().ConfigureAwait(false);
await _observers.PreRetry(retryContext).ConfigureAwait(false);
try
{
await next.Send(retryContext.Context).ConfigureAwait(false);
return;
}
catch (OperationCanceledException exception)
when(exception.CancellationToken == retryContext.CancellationToken || exception.CancellationToken == context.CancellationToken)
{
throw;
}
catch (Exception exception)
{
if (context.CancellationToken.IsCancellationRequested)
{
context.CancellationToken.ThrowIfCancellationRequested();
}
if (retryContext.Context.TryGetPayload(out RetryContext <TContext> payloadRetryContext))
{
if (_retryPolicy.IsHandled(exception))
{
await retryContext.RetryFaulted(exception).ConfigureAwait(false);
await _observers.RetryFault(payloadRetryContext).ConfigureAwait(false);
}
context.GetOrAddPayload(() => payloadRetryContext);
throw;
}
if (retryContext.Context.TryGetPayload(out RetryContext genericRetryContext))
{
if (_retryPolicy.IsHandled(exception))
{
await retryContext.RetryFaulted(exception).ConfigureAwait(false);
await _observers.RetryFault(genericRetryContext).ConfigureAwait(false);
}
context.GetOrAddPayload(() => genericRetryContext);
throw;
}
if (!retryContext.CanRetry(exception, out RetryContext <TContext> nextRetryContext))
{
if (_retryPolicy.IsHandled(exception))
{
await nextRetryContext.RetryFaulted(exception).ConfigureAwait(false);
await _observers.RetryFault(nextRetryContext).ConfigureAwait(false);
context.GetOrAddPayload(() => nextRetryContext);
}
throw;
}
await _observers.PostFault(nextRetryContext).ConfigureAwait(false);
retryContext = nextRetryContext;
}
}
context.CancellationToken.ThrowIfCancellationRequested();
}
public static async Task Retry(this IHostConfiguration hostConfiguration, Func <Task> factory, ISupervisor supervisor,
CancellationToken cancellationToken)
{
var description = hostConfiguration.HostAddress;
using var tokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, supervisor.Stopping);
var stoppingContext = new SupervisorStoppingContext(tokenSource.Token);
RetryPolicyContext <SupervisorStoppingContext> policyContext = hostConfiguration.ReceiveTransportRetryPolicy.CreatePolicyContext(stoppingContext);
try
{
RetryContext <SupervisorStoppingContext> retryContext = null;
while (!tokenSource.Token.IsCancellationRequested)
{
try
{
if (retryContext?.Delay != null)
{
await Task.Delay(retryContext.Delay.Value, tokenSource.Token).ConfigureAwait(false);
}
if (tokenSource.Token.IsCancellationRequested)
{
throw new ConnectionException($"The connection is stopping and cannot be used: {description}", retryContext?.Exception);
}
await factory().ConfigureAwait(false);
return;
}
catch (OperationCanceledException)
{
throw;
}
catch (Exception exception)
{
if (retryContext != null)
{
retryContext = retryContext.CanRetry(exception, out RetryContext <SupervisorStoppingContext> nextRetryContext)
? nextRetryContext
: null;
}
if (retryContext == null && !policyContext.CanRetry(exception, out retryContext))
{
throw;
}
}
if (tokenSource.Token.IsCancellationRequested)
{
break;
}
try
{
await Task.Delay(TimeSpan.FromSeconds(1), tokenSource.Token).ConfigureAwait(false);
}
catch
{
// just a little breather before reconnecting the receive transport
}
}
throw new ConnectionException($"The connection is stopping and cannot be used: {description}", retryContext?.Exception);
}
finally
{
policyContext.Dispose();
}
}
async Task Run()
{
var stoppingContext = new TransportStoppingContext(Stopping);
RetryPolicyContext <TransportStoppingContext> policyContext = _retryPolicy.CreatePolicyContext(stoppingContext);
try
{
RetryContext <TransportStoppingContext> retryContext = null;
while (!IsStopping)
{
try
{
if (retryContext?.Delay != null)
{
await Task.Delay(retryContext.Delay.Value, Stopping).ConfigureAwait(false);
}
if (!IsStopping)
{
await RunTransport().ConfigureAwait(false);
}
}
catch (OperationCanceledException exception)
{
if (retryContext == null)
{
await NotifyFaulted(exception).ConfigureAwait(false);
}
throw;
}
catch (Exception exception)
{
if (retryContext != null)
{
retryContext = retryContext.CanRetry(exception, out RetryContext <TransportStoppingContext> nextRetryContext)
? nextRetryContext
: null;
}
if (retryContext == null && !policyContext.CanRetry(exception, out retryContext))
{
break;
}
}
if (IsStopping)
{
break;
}
try
{
await Task.Delay(TimeSpan.FromSeconds(1), Stopping).ConfigureAwait(false);
}
catch
{
// just a little breather before reconnecting the receive transport
}
}
}
catch (OperationCanceledException exception)
{
if (exception.CancellationToken != Stopping)
{
LogContext.Debug?.Log(exception, "ReceiveTransport Operation Cancelled: {InputAddress}", _context.InputAddress);
}
}
catch (Exception exception)
{
LogContext.Debug?.Log(exception, "ReceiveTransport Run Exception: {InputAddress}", _context.InputAddress);
}
finally
{
policyContext.Dispose();
}
}
