public Task Send(ConsumeContext <TData> context)
{
using (RetryPolicyContext <ConsumeContext <TData> > policyContext = _retryPolicy.CreatePolicyContext(context))
{
var exception = new SagaException("An existing saga instance was not found", typeof(TInstance), typeof(TData), context.CorrelationId ?? Guid
.Empty);
if (!policyContext.CanRetry(exception, out RetryContext <ConsumeContext <TData> > retryContext))
{
return(_finalPipe.Send(context));
}
int previousDeliveryCount = context.GetRedeliveryCount();
for (int retryIndex = 0; retryIndex < previousDeliveryCount; retryIndex++)
{
if (!retryContext.CanRetry(exception, out retryContext))
{
return(_finalPipe.Send(context));
}
}
var schedulerContext = context.GetPayload <MessageSchedulerContext>();
MessageRedeliveryContext redeliveryContext = new ScheduleMessageRedeliveryContext <TData>(context, schedulerContext);
var delay = retryContext.Delay ?? TimeSpan.Zero;
return(redeliveryContext.ScheduleRedelivery(delay));
}
}
public static async Task Retry <T>(this IRetryPolicy retryPolicy, ConsumeContext <T> context,
Func <ConsumeContext <T>, Task> retryMethod)
where T : class
{
using (RetryPolicyContext <ConsumeContext <T> > policyContext = retryPolicy.CreatePolicyContext(context))
{
try
{
await retryMethod(policyContext.Context).ConfigureAwait(false);
}
catch (Exception exception)
{
if (context.TryGetPayload(out RetryContext <ConsumeContext <T> > retryContext))
{
throw;
}
if (!policyContext.CanRetry(exception, out retryContext))
{
context.GetOrAddPayload(() => retryContext);
throw;
}
await Attempt(retryContext, retryMethod).ConfigureAwait(false);
}
}
}
public static async Task Retry(this IRetryPolicy retryPolicy, Func <Task> retryMethod, CancellationToken cancellationToken = default)
{
var inlinePipeContext = new InlinePipeContext(cancellationToken);
using (RetryPolicyContext <InlinePipeContext> policyContext = retryPolicy.CreatePolicyContext(inlinePipeContext))
{
try
{
await retryMethod().ConfigureAwait(false);
}
catch (OperationCanceledException)
{
throw;
}
catch (Exception exception)
{
if (!policyContext.CanRetry(exception, out RetryContext <InlinePipeContext> retryContext))
{
throw;
}
try
{
await Attempt(inlinePipeContext, retryContext, retryMethod).ConfigureAwait(false);
}
catch (OperationCanceledException ex) when(ex.CancellationToken == cancellationToken)
{
}
throw;
}
}
}
public static async Task <T> Retry <T>(this IRetryPolicy retryPolicy, Func <Task <T> > retryMethod,
CancellationToken cancellationToken = default(CancellationToken))
{
var inlinePipeContext = new InlinePipeContext(cancellationToken);
using (RetryPolicyContext <InlinePipeContext> policyContext = retryPolicy.CreatePolicyContext(inlinePipeContext))
{
try
{
return(await retryMethod().ConfigureAwait(false));
}
catch (Exception exception)
{
if (cancellationToken.IsCancellationRequested)
{
if (exception is OperationCanceledException canceledException &&
canceledException.CancellationToken == cancellationToken)
{
throw;
}
cancellationToken.ThrowIfCancellationRequested();
}
if (!policyContext.CanRetry(exception, out RetryContext <InlinePipeContext> retryContext))
{
throw;
}
return(await Attempt(inlinePipeContext, retryContext, retryMethod).ConfigureAwait(false));
}
}
}
public static async Task Retry(this IRetryPolicy retryPolicy, Func <Task> retryMethod, CancellationToken cancellationToken = default(CancellationToken))
{
RetryPolicyContext <InlinePipeContext> policyContext = retryPolicy.CreatePolicyContext(new InlinePipeContext(cancellationToken));
try
{
await retryMethod().ConfigureAwait(false);
}
catch (Exception exception)
{
if (cancellationToken.IsCancellationRequested)
{
var canceledException = exception as OperationCanceledException;
if (canceledException != null && canceledException.CancellationToken == cancellationToken)
{
throw;
}
cancellationToken.ThrowIfCancellationRequested();
}
RetryContext <InlinePipeContext> retryContext;
if (!policyContext.CanRetry(exception, out retryContext))
{
throw;
}
await Attempt(retryContext, retryMethod).ConfigureAwait(false);
}
}
async Task IFilter <TContext> .Send(TContext context, IPipe <TContext> next)
{
RetryPolicyContext <TContext> policyContext = _retryPolicy.CreatePolicyContext(context);
try
{
await next.Send(policyContext.Context).ConfigureAwait(false);
}
catch (Exception exception)
{
RetryContext <TContext> payloadRetryContext;
if (context.TryGetPayload(out payloadRetryContext) && !payloadRetryContext.CanRetry(exception, out payloadRetryContext))
{
await policyContext.RetryFaulted(exception).ConfigureAwait(false);
throw;
}
RetryContext <TContext> retryContext;
if (!policyContext.CanRetry(exception, out retryContext))
{
await retryContext.RetryFaulted(exception).ConfigureAwait(false);
context.GetOrAddPayload(() => retryContext);
throw;
}
await Attempt(retryContext, next).ConfigureAwait(false);
}
}
public bool CanRetry(Exception exception, out RetryContext <CommandContext> retryContext)
{
var canRetry = _policyContext.CanRetry(exception, out RetryContext <CommandContext> policyRetryContext);
retryContext = new ConsumeContextRetryContext(policyRetryContext, canRetry ? _context.CreateNext() : _context);
return(canRetry);
}
async Task Run()
{
var stoppingContext = new TransportStoppingContext(Stopping);
RetryPolicyContext <TransportStoppingContext> policyContext = _retryPolicy.CreatePolicyContext(stoppingContext);
try
{
while (!IsStopping)
{
RetryContext <TransportStoppingContext> retryContext = null;
try
{
if (retryContext != null)
{
LogContext.Warning?.Log(retryContext.Exception, "Retrying {Delay}: {Message}", retryContext.Delay,
retryContext.Exception.Message);
if (retryContext.Delay.HasValue)
{
await Task.Delay(retryContext.Delay.Value, Stopping).ConfigureAwait(false);
}
}
if (!IsStopping)
{
await RunTransport().ConfigureAwait(false);
}
}
catch (OperationCanceledException)
{
throw;
}
catch (Exception exception)
{
if (retryContext != null)
{
retryContext = retryContext.CanRetry(exception, out RetryContext <TransportStoppingContext> nextRetryContext)
? nextRetryContext
: null;
}
if (retryContext == null && !policyContext.CanRetry(exception, out retryContext))
{
break;
}
}
}
}
catch (Exception exception)
{
LogContext.Debug?.Log(exception, "ReceiveTransport Run Exception: {InputAddress}", _context.InputAddress);
}
finally
{
policyContext.Dispose();
}
}
public bool CanRetry(Exception exception, out RetryContext <ConsumeContext> retryContext)
{
RetryContext <ConsumeContext> policyRetryContext;
var canRetry = _policyContext.CanRetry(exception, out policyRetryContext);
_context.LogRetry(exception);
retryContext = new ConsumeContextRetryContext(policyRetryContext, _context);
return(canRetry);
}
async Task IFilter <TContext> .Send(TContext context, IPipe <TContext> next)
{
RetryPolicyContext <TContext> policyContext = _retryPolicy.CreatePolicyContext(context);
await _observers.PostCreate(policyContext).ConfigureAwait(false);
try
{
await next.Send(policyContext.Context).ConfigureAwait(false);
}
catch (Exception exception)
{
RetryContext <TContext> payloadRetryContext;
if (context.TryGetPayload(out payloadRetryContext))
{
await policyContext.RetryFaulted(exception).ConfigureAwait(false);
await _observers.RetryFault(payloadRetryContext).ConfigureAwait(false);
throw;
}
RetryContext genericRetryContext;
if (context.TryGetPayload(out genericRetryContext))
{
await policyContext.RetryFaulted(exception).ConfigureAwait(false);
await _observers.RetryFault(genericRetryContext).ConfigureAwait(false);
throw;
}
RetryContext <TContext> retryContext;
if (!policyContext.CanRetry(exception, out retryContext))
{
await retryContext.RetryFaulted(exception).ConfigureAwait(false);
await _observers.RetryFault(retryContext).ConfigureAwait(false);
context.GetOrAddPayload(() => retryContext);
throw;
}
await _observers.PostFault(retryContext).ConfigureAwait(false);
if (retryContext.Delay.HasValue)
{
await Task.Delay(retryContext.Delay.Value).ConfigureAwait(false);
}
await Attempt(retryContext, next).ConfigureAwait(false);
}
}
public bool CanRetry(Exception exception, out RetryContext<ConsumeContext> retryContext)
{
var canRetry = _policyContext.CanRetry(exception, out RetryContext<ConsumeContext> policyRetryContext);
if (canRetry)
{
_context.LogRetry(exception);
_registration = _cancellationToken.Register(Cancel);
}
retryContext = new ConsumeContextRetryContext(policyRetryContext, canRetry ? _context.CreateNext(policyRetryContext) : _context);
return canRetry;
}
public static async Task Retry(this IRetryPolicy retryPolicy, Func <Task> retryMethod, CancellationToken cancellationToken = default(CancellationToken))
{
RetryPolicyContext <InlinePipeContext> policyContext = retryPolicy.CreatePolicyContext(new InlinePipeContext());
try
{
await retryMethod().ConfigureAwait(false);
}
catch (Exception exception)
{
RetryContext <InlinePipeContext> retryContext;
if (!policyContext.CanRetry(exception, out retryContext))
{
throw;
}
await Attempt(retryContext, retryMethod, cancellationToken).ConfigureAwait(false);
}
}
async Task IFilter <TContext> .Send(TContext context, IPipe <TContext> next)
{
using (RetryPolicyContext <TContext> policyContext = _retryPolicy.CreatePolicyContext(context))
{
await _observers.PostCreate(policyContext).ConfigureAwait(false);
try
{
await next.Send(policyContext.Context).ConfigureAwait(false);
}
catch (OperationCanceledException exception)
when(exception.CancellationToken == policyContext.Context.CancellationToken || exception.CancellationToken == context.CancellationToken)
{
throw;
}
catch (Exception exception)
{
if (policyContext.Context.CancellationToken.IsCancellationRequested)
{
policyContext.Context.CancellationToken.ThrowIfCancellationRequested();
}
if (policyContext.Context.TryGetPayload(out RetryContext <TContext> payloadRetryContext))
{
if (_retryPolicy.IsHandled(exception))
{
await policyContext.RetryFaulted(exception).ConfigureAwait(false);
await _observers.RetryFault(payloadRetryContext).ConfigureAwait(false);
}
context.GetOrAddPayload(() => payloadRetryContext);
throw;
}
if (policyContext.Context.TryGetPayload(out RetryContext genericRetryContext))
{
if (_retryPolicy.IsHandled(exception))
{
await policyContext.RetryFaulted(exception).ConfigureAwait(false);
await _observers.RetryFault(genericRetryContext).ConfigureAwait(false);
}
context.GetOrAddPayload(() => genericRetryContext);
throw;
}
if (!policyContext.CanRetry(exception, out RetryContext <TContext> retryContext))
{
if (_retryPolicy.IsHandled(exception))
{
await retryContext.RetryFaulted(exception).ConfigureAwait(false);
await _observers.RetryFault(retryContext).ConfigureAwait(false);
context.GetOrAddPayload(() => retryContext);
}
throw;
}
await _observers.PostFault(retryContext).ConfigureAwait(false);
await Attempt(context, retryContext, next).ConfigureAwait(false);
}
}
}
public async Task Send(ConsumeContext <T> context, IPipe <ConsumeContext <T> > next)
{
RetryPolicyContext <ConsumeContext <T> > policyContext = _retryPolicy.CreatePolicyContext(context);
await _observers.PostCreate(policyContext).ConfigureAwait(false);
try
{
await next.Send(policyContext.Context).ConfigureAwait(false);
}
catch (Exception exception)
{
if (context.CancellationToken.IsCancellationRequested)
{
if (exception is OperationCanceledException canceledException && canceledException.CancellationToken == context.CancellationToken)
{
throw;
}
context.CancellationToken.ThrowIfCancellationRequested();
}
if (policyContext.Context.TryGetPayload(out RetryContext <ConsumeContext <T> > payloadRetryContext))
{
await policyContext.RetryFaulted(exception).ConfigureAwait(false);
await _observers.RetryFault(payloadRetryContext).ConfigureAwait(false);
context.GetOrAddPayload(() => payloadRetryContext);
throw;
}
if (policyContext.Context.TryGetPayload(out RetryContext genericRetryContext))
{
await policyContext.RetryFaulted(exception).ConfigureAwait(false);
await _observers.RetryFault(genericRetryContext).ConfigureAwait(false);
context.GetOrAddPayload(() => genericRetryContext);
throw;
}
if (!policyContext.CanRetry(exception, out RetryContext <ConsumeContext <T> > retryContext))
{
await retryContext.RetryFaulted(exception).ConfigureAwait(false);
await _observers.RetryFault(retryContext).ConfigureAwait(false);
if (_retryPolicy.IsHandled(exception))
{
context.GetOrAddPayload(() => retryContext);
}
throw;
}
int previousDeliveryCount = context.Headers.Get(MessageHeaders.RedeliveryCount, default(int?)) ?? 0;
for (int retryIndex = 0; retryIndex < previousDeliveryCount; retryIndex++)
{
if (!retryContext.CanRetry(exception, out retryContext))
{
await retryContext.RetryFaulted(exception).ConfigureAwait(false);
await _observers.RetryFault(retryContext).ConfigureAwait(false);
if (_retryPolicy.IsHandled(exception))
{
context.GetOrAddPayload(() => retryContext);
}
throw;
}
}
await _observers.PostFault(retryContext).ConfigureAwait(false);
try
{
if (!context.TryGetPayload(out MessageRedeliveryContext redeliveryContext))
{
throw new ContextException("The message redelivery context was not available to delay the message", exception);
}
var delay = retryContext.Delay ?? TimeSpan.Zero;
await redeliveryContext.ScheduleRedelivery(delay).ConfigureAwait(false);
await context.NotifyConsumed(context, context.ReceiveContext.ElapsedTime, TypeMetadataCache <RedeliveryRetryFilter <T> > .ShortName).ConfigureAwait(false);
}
catch (Exception redeliveryException)
{
throw new ContextException("The message delivery could not be rescheduled", new AggregateException(redeliveryException, exception));
}
}
}
async Task Run()
{
var stoppingContext = new TransportStoppingContext(Stopping);
RetryPolicyContext <TransportStoppingContext> policyContext = _retryPolicy.CreatePolicyContext(stoppingContext);
try
{
RetryContext <TransportStoppingContext> retryContext = null;
while (!IsStopping)
{
try
{
if (retryContext?.Delay != null)
{
await Task.Delay(retryContext.Delay.Value, Stopping).ConfigureAwait(false);
}
if (!IsStopping)
{
await RunTransport().ConfigureAwait(false);
}
}
catch (OperationCanceledException exception)
{
if (retryContext == null)
{
await NotifyFaulted(exception).ConfigureAwait(false);
}
throw;
}
catch (Exception exception)
{
if (retryContext != null)
{
retryContext = retryContext.CanRetry(exception, out RetryContext <TransportStoppingContext> nextRetryContext)
? nextRetryContext
: null;
}
if (retryContext == null && !policyContext.CanRetry(exception, out retryContext))
{
break;
}
}
if (IsStopping)
{
break;
}
try
{
await Task.Delay(TimeSpan.FromSeconds(1), Stopping).ConfigureAwait(false);
}
catch
{
// just a little breather before reconnecting the receive transport
}
}
}
catch (OperationCanceledException exception)
{
if (exception.CancellationToken != Stopping)
{
LogContext.Debug?.Log(exception, "ReceiveTransport Operation Cancelled: {InputAddress}", _context.InputAddress);
}
}
catch (Exception exception)
{
LogContext.Debug?.Log(exception, "ReceiveTransport Run Exception: {InputAddress}", _context.InputAddress);
}
finally
{
policyContext.Dispose();
}
}
public async Task Send(TContext context, IPipe <TContext> next)
{
using (RetryPolicyContext <TContext> policyContext = _retryPolicy.CreatePolicyContext(context))
{
if (_observers.Count > 0)
{
await _observers.PostCreate(policyContext).ConfigureAwait(false);
}
try
{
await next.Send(policyContext.Context).ConfigureAwait(false);
}
catch (OperationCanceledException exception)
when(exception.CancellationToken == context.CancellationToken || exception.CancellationToken == policyContext.Context.CancellationToken)
{
throw;
}
catch (Exception exception)
{
if (policyContext.Context.CancellationToken.IsCancellationRequested)
{
policyContext.Context.CancellationToken.ThrowIfCancellationRequested();
}
if (!policyContext.CanRetry(exception, out RetryContext <TContext> retryContext))
{
if (_retryPolicy.IsHandled(exception))
{
context.GetOrAddPayload(() => retryContext);
await retryContext.RetryFaulted(exception).ConfigureAwait(false);
if (_observers.Count > 0)
{
await _observers.RetryFault(retryContext).ConfigureAwait(false);
}
}
throw;
}
var previousDeliveryCount = context.GetRedeliveryCount();
for (var retryIndex = 0; retryIndex < previousDeliveryCount; retryIndex++)
{
if (!retryContext.CanRetry(exception, out retryContext))
{
if (_retryPolicy.IsHandled(exception))
{
context.GetOrAddPayload(() => retryContext);
await retryContext.RetryFaulted(exception).ConfigureAwait(false);
if (_observers.Count > 0)
{
await _observers.RetryFault(retryContext).ConfigureAwait(false);
}
}
throw;
}
}
if (_observers.Count > 0)
{
await _observers.PostFault(retryContext).ConfigureAwait(false);
}
try
{
var redeliveryContext = context.GetPayload <MessageRedeliveryContext>();
var delay = retryContext.Delay ?? TimeSpan.Zero;
await redeliveryContext.ScheduleRedelivery(delay).ConfigureAwait(false);
await context.NotifyConsumed(context, context.ReceiveContext.ElapsedTime,
TypeMetadataCache <RedeliveryRetryFilter <TContext, TMessage> > .ShortName).ConfigureAwait(false);
}
catch (Exception redeliveryException)
{
throw new TransportException(context.ReceiveContext.InputAddress, "The message delivery could not be rescheduled",
new AggregateException(redeliveryException, exception));
}
}
}
}
public static async Task Retry(this IHostConfiguration hostConfiguration, Func <Task> factory, ISupervisor supervisor,
CancellationToken cancellationToken)
{
var description = hostConfiguration.HostAddress;
using var tokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, supervisor.Stopping);
var stoppingContext = new SupervisorStoppingContext(tokenSource.Token);
RetryPolicyContext <SupervisorStoppingContext> policyContext = hostConfiguration.ReceiveTransportRetryPolicy.CreatePolicyContext(stoppingContext);
try
{
RetryContext <SupervisorStoppingContext> retryContext = null;
while (!tokenSource.Token.IsCancellationRequested)
{
try
{
if (retryContext?.Delay != null)
{
await Task.Delay(retryContext.Delay.Value, tokenSource.Token).ConfigureAwait(false);
}
if (tokenSource.Token.IsCancellationRequested)
{
throw new ConnectionException($"The connection is stopping and cannot be used: {description}", retryContext?.Exception);
}
await factory().ConfigureAwait(false);
return;
}
catch (OperationCanceledException)
{
throw;
}
catch (Exception exception)
{
if (retryContext != null)
{
retryContext = retryContext.CanRetry(exception, out RetryContext <SupervisorStoppingContext> nextRetryContext)
? nextRetryContext
: null;
}
if (retryContext == null && !policyContext.CanRetry(exception, out retryContext))
{
throw;
}
}
if (tokenSource.Token.IsCancellationRequested)
{
break;
}
try
{
await Task.Delay(TimeSpan.FromSeconds(1), tokenSource.Token).ConfigureAwait(false);
}
catch
{
// just a little breather before reconnecting the receive transport
}
}
throw new ConnectionException($"The connection is stopping and cannot be used: {description}", retryContext?.Exception);
}
finally
{
policyContext.Dispose();
}
}
public async Task Send(ConsumeContext <T> context, IPipe <ConsumeContext <T> > next)
{
RetryPolicyContext <ConsumeContext <T> > policyContext = _retryPolicy.CreatePolicyContext(context);
await _observers.PostCreate(policyContext).ConfigureAwait(false);
try
{
await next.Send(policyContext.Context).ConfigureAwait(false);
}
catch (Exception exception)
{
RetryContext <ConsumeContext <T> > payloadRetryContext;
if (context.TryGetPayload(out payloadRetryContext))
{
await policyContext.RetryFaulted(exception).ConfigureAwait(false);
await _observers.RetryFault(payloadRetryContext).ConfigureAwait(false);
throw;
}
RetryContext genericRetryContext;
if (context.TryGetPayload(out genericRetryContext))
{
await policyContext.RetryFaulted(exception).ConfigureAwait(false);
await _observers.RetryFault(genericRetryContext).ConfigureAwait(false);
throw;
}
RetryContext <ConsumeContext <T> > retryContext;
if (!policyContext.CanRetry(exception, out retryContext))
{
await retryContext.RetryFaulted(exception).ConfigureAwait(false);
await _observers.RetryFault(retryContext).ConfigureAwait(false);
context.GetOrAddPayload(() => retryContext);
throw;
}
int previousDeliveryCount = context.Headers.Get(MessageHeaders.RedeliveryCount, default(int?)) ?? 0;
for (int retryIndex = 0; retryIndex < previousDeliveryCount; retryIndex++)
{
if (!retryContext.CanRetry(exception, out retryContext))
{
await retryContext.RetryFaulted(exception).ConfigureAwait(false);
await _observers.RetryFault(retryContext).ConfigureAwait(false);
context.GetOrAddPayload(() => retryContext);
throw;
}
}
await _observers.PostFault(retryContext).ConfigureAwait(false);
try
{
MessageRedeliveryContext redeliveryContext;
if (!context.TryGetPayload(out redeliveryContext))
{
throw new ContextException("The message redelivery context was not available to delay the message", exception);
}
var delay = retryContext.Delay ?? TimeSpan.Zero;
await redeliveryContext.ScheduleRedelivery(delay).ConfigureAwait(false);
}
catch (Exception redeliveryException)
{
throw new ContextException("The message delivery could not be rescheduled", new AggregateException(redeliveryException, exception));
}
}
}
async Task RunJob(PipeContext context, IJobConsumer <TJob> jobConsumer)
{
var jobContext = context.GetPayload <JobContext <TJob> >();
RetryPolicyContext <JobContext <TJob> > policyContext = _retryPolicy.CreatePolicyContext(jobContext);
try
{
await jobContext.NotifyStarted().ConfigureAwait(false);
await jobConsumer.Run(jobContext).ConfigureAwait(false);
await jobContext.NotifyCompleted().ConfigureAwait(false);
}
catch (OperationCanceledException exception)
{
if (jobContext.CancellationToken == exception.CancellationToken)
{
await jobContext.NotifyCanceled("Operation canceled").ConfigureAwait(false);
}
}
catch (Exception exception)
{
if (!policyContext.CanRetry(exception, out RetryContext <JobContext <TJob> > retryContext))
{
if (_retryPolicy.IsHandled(exception))
{
context.GetOrAddPayload(() => retryContext);
await retryContext.RetryFaulted(exception).ConfigureAwait(false);
}
await jobContext.NotifyFaulted(exception).ConfigureAwait(false);
return;
}
var currentRetryAttempt = jobContext.RetryAttempt;
for (var retryIndex = 0; retryIndex < currentRetryAttempt; retryIndex++)
{
if (!retryContext.CanRetry(exception, out retryContext))
{
if (_retryPolicy.IsHandled(exception))
{
context.GetOrAddPayload(() => retryContext);
await retryContext.RetryFaulted(exception).ConfigureAwait(false);
}
await jobContext.NotifyFaulted(exception).ConfigureAwait(false);
return;
}
}
var delay = retryContext.Delay ?? TimeSpan.Zero;
await jobContext.NotifyFaulted(exception, delay).ConfigureAwait(false);
}
finally
{
policyContext.Dispose();
}
}