private int _getRecoveryCount(Message message, RecoverabilityContext context)
{
var count = message.UserProperties.ContainsKey("RecoveryCount") ?
(int)message.UserProperties["RecoveryCount"]
: 0;
return(count);
}
public async Task <RecoverabilityContext> OnPreHandle(RecoverabilityContext context)
{
if (IsControlMessage(context.Message))//never try to reschedule the control messages themselves
{
return(await _handleControlMessage(context.Message, context));
}
return(context);
}
public async Task <RecoverabilityContext> OnPreHandle(RecoverabilityContext context)
{
var count = _getRecoveryCount(context.Message, context);
if (await _handleMaxRetry(count, context.Message, context))
{
context.SkipMessage = true;
}
return(context);
}
public async Task OnPostHandle(RecoverabilityContext context)// int timesQueued, Endpoint endpoint, IMessageReceiver receiver, Message controlMessage = null, string description = null, Exception exc = null)
{
try
{
if (context.TempData.ContainsKey("ControlMessage"))
{
await _completeImmediateRetryPolicy.ExecuteAsync(() =>
context.Receiver.CompleteAsync(((Message)context.TempData["ControlMessage"]).SystemProperties.LockToken)
);
}
}
catch (Exception exc)
{
///TODO log that control message couldn't be completed
}
}
private async Task _handleMessage(RecoverabilityContext context, CancellationToken cancellationToken)
{
bool isHandled = false;
try
{
context = await _recoverability.OnPreHandle(context);
if (context.SkipMessage)
{
return;
}
await _handleMessageFunc(
new TransportMessage(context.Message.Label, context.Message.Body, _metaDataMapper.ExtractMetaData(context.Message)),
_endpoint
);
isHandled = true;
}
catch (NonTransientException nte)
{
_logger.LogError(nte, "AzureServiceBusMessagePump encountered a NonTransient Exception handling a message. Sending to DLQ. Endpoint: {EndpointName}", context.Endpoint.Name);
await _receiver.DeadLetterAsync(context.Message.SystemProperties.LockToken);
}
catch (Exception exc)
{
//backoff retry
_logger.LogError(exc, "AzureServiceBusMessagePump encountered an exception handling a message. Attempting to recover. Endpoint: {EndpointName}. Label: {messageLabel}", context.Endpoint.Name, context.Message?.Label);
await _recoverability.Recover(context);
}
//if an exception occurs completing the message then it will stay on the queue and be handled again
//we don't want to trigger recoverability/retry
if (isHandled)
{
await _completeImmediateRetryPolicy.ExecuteAsync(() =>
_receiver.CompleteAsync(context.Message.SystemProperties.LockToken)
);
await _recoverability.OnPostHandle(context);
}
}
public async Task Recover(RecoverabilityContext context)
{
var sender = AzureServiceBusClientCache.GetSender(context.Endpoint.Settings.ConnectionString);
var count = _getRecoveryCount(context.Message, context);
var retryMessage = context.Message.Clone();
retryMessage.MessageId = Guid.NewGuid().ToString();
retryMessage.UserProperties.Add("OriginalMessageId", context.Message.MessageId);
retryMessage.UserProperties.Add("RecoveryCount", ++count);
await sender.ScheduleMessageAsync(
retryMessage,
this._retryStrategy.GetNextDateUtc(++count)
);
//complete the original message - we've already scheduled a clone
await _completeImmediateRetryPolicy.ExecuteAsync(() =>
context.Receiver.CompleteAsync(context.Message.SystemProperties.LockToken)
);
}
public async Task Recover(RecoverabilityContext context) // int timesQueued, Endpoint endpoint, IMessageReceiver receiver, Message controlMessage = null, string description = null, Exception exc = null)
{
if (IsControlMessage(context.Message)) //never try to reschedule the control messages themselves
{
return;
}
var count = context.TempData.ContainsKey("ControlMessageContent") ?
((RecoverabilityControlMessage)context.TempData["ControlMessageContent"]).RecoveryCount
: 0;
var sender = AzureServiceBusClientCache.GetSender(context.Endpoint.Settings.ConnectionString);
var retryMessage = new RecoverabilityControlMessage(context.Message.SystemProperties.SequenceNumber, ++count);
//schedule a special control message to be delivered in the future to tell the the deferred message to be retrieved and re processed
await sender.ScheduleMessageAsync(
_createServiceBusMessage(retryMessage),
this._retryStrategy.GetNextDateUtc(retryMessage.RecoveryCount)
);
//defer the current message / first time through
if (!context.TempData.ContainsKey("ControlMessage"))
{
await context.Receiver.DeferAsync(context.Message.SystemProperties.LockToken);
}
else //already deferred. complete the control message / we just sent a new one above
{
await _completeImmediateRetryPolicy.ExecuteAsync(() =>
context.Receiver.CompleteAsync(((Message)context.TempData["ControlMessage"]).SystemProperties.LockToken)
);
//release the lock on the current deferred message
await _completeImmediateRetryPolicy.ExecuteAsync(() =>
context.Receiver.AbandonAsync(context.Message.SystemProperties.LockToken)
);
}
}
private async Task <bool> _handleMaxRetry(int recoveryCount, Message message, RecoverabilityContext context)
{
if (message.SystemProperties.DeliveryCount > 1 || recoveryCount > _maxDeliveryCount)
{
await context.Receiver.DeadLetterAsync(message.SystemProperties.LockToken);
return(true);
}
return(false);
}
public Task OnPostHandle(RecoverabilityContext context) // int timesQueued, Endpoint endpoint, IMessageReceiver receiver, Message controlMessage = null, string description = null, Exception exc = null)
{
//no cleanup to do here
return(Task.CompletedTask);
}
private async Task <RecoverabilityContext> _handleControlMessage(Message serviceBusControlMessage, RecoverabilityContext context)
{
try
{
var json = Encoding.UTF8.GetString(serviceBusControlMessage.Body);
var controlMessage = (RecoverabilityControlMessage)JsonConvert.DeserializeObject <RecoverabilityControlMessage>(json);
context.Message = await _receiveDeferredImmediateRetryPolicy.ExecuteAsync(() =>
context.Receiver.ReceiveDeferredMessageAsync(controlMessage.SequenceNumber)
);
context.TempData["ControlMessage"] = serviceBusControlMessage;
context.TempData["ControlMessageContent"] = controlMessage;
return(context);
}
catch (Exception exc)
{
//wrap all exceptions in a nontransient exception because retry won't help us now
///TODO: if deferred message times-out then it will throw exceptions until the control message retries expire
///need to use poly to retry and circuit break
throw new NonTransientException("Error handling Recoverability control message", exc);
}
}