// RabbitMQ retry that will publish a message to the retry queue.
// Keep in mind that it might get processed by the other services using the same routing key and wildcards.
private async Task <Acknowledgement> TryHandleWithRequeuingAsync <TMessage>(TMessage message,
CorrelationContext correlationContext,
Func <Task> handle, Func <TMessage, S3Exception, IRejectedEvent> onError = null)
{
var messageName = message.GetType().Name;
var retryMessage = correlationContext.Retries == 0
? string.Empty
: $"Retry: {correlationContext.Retries}'.";
_logger.LogInformation($"Handling a message: '{messageName}' " +
$"with correlation id: '{correlationContext.Id}'. {retryMessage}");
try
{
await handle();
_logger.LogInformation($"Handled a message: '{messageName}' " +
$"with correlation id: '{correlationContext.Id}'. {retryMessage}");
return(new Ack());
}
catch (Exception exception)
{
_logger.LogError(exception, exception.Message);
if (exception is S3Exception s3Exception && onError != null)
{
var rejectedEvent = onError(message, s3Exception);
await _busClient.PublishAsync(rejectedEvent, ctx => ctx.UseMessageContext(correlationContext));
_logger.LogInformation($"Published a rejected event: '{rejectedEvent.GetType().Name}' " +
$"for the message: '{messageName}' with correlation id: '{correlationContext.Id}'.");
return(new Ack());
}
if (correlationContext.Retries >= _retries)
{
await _busClient.PublishAsync(RejectedEvent.For(messageName),
ctx => ctx.UseMessageContext(correlationContext));
throw new Exception($"Unable to handle a message: '{messageName}' " +
$"with correlation id: '{correlationContext.Id}' " +
$"after {correlationContext.Retries} retries.", exception);
}
_logger.LogInformation($"Unable to handle a message: '{messageName}' " +
$"with correlation id: '{correlationContext.Id}', " +
$"retry {correlationContext.Retries}/{_retries}...");
return(Retry.In(TimeSpan.FromSeconds(_retryInterval)));
}
}
// Internal retry for services that subscribe to the multiple events of the same types.
// It does not interfere with the routing keys and wildcards (see TryHandleWithRequeuingAsync() below).
private async Task <Acknowledgement> TryHandleAsync <TMessage>(TMessage message,
CorrelationContext correlationContext,
Func <Task> handle, Func <TMessage, S3Exception, IRejectedEvent> onError = null)
{
var currentRetry = 0;
var retryPolicy = Policy
.Handle <Exception>()
.WaitAndRetryAsync(_retries, i => TimeSpan.FromSeconds(_retryInterval));
var messageName = message.GetType().Name;
return(await retryPolicy.ExecuteAsync <Acknowledgement>(async() =>
{
var scope = _tracer
.BuildSpan("executing-handler")
.AsChildOf(_tracer.ActiveSpan)
.StartActive(true);
using (scope)
{
var span = scope.Span;
try
{
var retryMessage = currentRetry == 0
? string.Empty
: $"Retry: {currentRetry}'.";
var preLogMessage = $"Handling a message: '{messageName}' " +
$"with correlation id: '{correlationContext.Id}'. {retryMessage}";
_logger.LogInformation(preLogMessage);
span.Log(preLogMessage);
await handle();
var postLogMessage = $"Handled a message: '{messageName}' " +
$"with correlation id: '{correlationContext.Id}'. {retryMessage}";
_logger.LogInformation(postLogMessage);
span.Log(postLogMessage);
return new Ack();
}
catch (Exception exception)
{
currentRetry++;
_logger.LogError(exception, exception.Message);
span.Log(exception.Message);
span.SetTag(Tags.Error, true);
if (exception is S3Exception s3Exception && onError != null)
{
var rejectedEvent = onError(message, s3Exception);
await _busClient.PublishAsync(rejectedEvent, ctx => ctx.UseMessageContext(correlationContext));
_logger.LogInformation($"Published a rejected event: '{rejectedEvent.GetType().Name}' " +
$"for the message: '{messageName}' with correlation id: '{correlationContext.Id}'.");
span.SetTag("error-type", "domain");
return new Ack();
}
span.SetTag("error-type", "infrastructure");
throw new Exception($"Unable to handle a message: '{messageName}' " +
$"with correlation id: '{correlationContext.Id}', " +
$"retry {currentRetry - 1}/{_retries}...");
}
}
}));
}
