public Task Init(Func <MessageContext, Task> pump, Func <ErrorContext, Task <ErrorHandleResult> > onError, CriticalError criticalError, PushSettings pushSettings)
{
topologyOperator = DetermineTopologyOperator(pushSettings.InputQueue);
messagePump = pump;
var name = $"MessagePump on the queue `{pushSettings.InputQueue}`";
circuitBreaker = new RepeatedFailuresOverTimeCircuitBreaker(name, timeToWaitBeforeTriggering, ex => criticalError.Raise("Failed to receive message from Azure Service Bus.", ex));
if (pushSettings.PurgeOnStartup)
{
throw new InvalidOperationException("Azure Service Bus transport doesn't support PurgeOnStartup behavior");
}
inputQueue = pushSettings.InputQueue;
topologyOperator.OnIncomingMessage(async(incoming, receiveContext) =>
{
if (circuitBreaker == null || throttler == null) /* can be disposed by fody injected logic, in theory */
{
return;
}
var tokenSource = new CancellationTokenSource();
receiveContext.CancellationToken = tokenSource.Token;
circuitBreaker.Success();
var transportTransaction = new TransportTransaction();
transportTransaction.Set(receiveContext);
await throttler.WaitAsync(receiveContext.CancellationToken).ConfigureAwait(false);
try
{
await messagePump(new MessageContext(incoming.MessageId, incoming.Headers, incoming.Body, transportTransaction, tokenSource, new ContextBag())).ConfigureAwait(false);
}
finally
{
throttler.Release();
}
});
topologyOperator.OnError(exception => circuitBreaker?.Failure(exception));
topologyOperator.OnProcessingFailure(onError);
topologyOperator.SetCriticalError(criticalError);
return(TaskEx.Completed);
}
async Task InnerProcessMessage(Task <Message> receiveTask)
{
Message message = null;
try
{
// Workaround for ASB MessageReceiver.Receive() that has a timeout and doesn't take a CancellationToken.
// We want to track how many receives are waiting and could be ignored when endpoint is stopping.
// TODO: remove workaround when https://github.com/Azure/azure-service-bus-dotnet/issues/439 is fixed
Interlocked.Increment(ref numberOfExecutingReceives);
message = await receiveTask.ConfigureAwait(false);
circuitBreaker.Success();
}
catch (ServiceBusException sbe) when(sbe.IsTransient)
{
}
catch (ObjectDisposedException)
{
// Can happen during endpoint shutdown
}
catch (Exception exception)
{
logger.WarnFormat("Failed to receive a message. Exception: {0}", exception.Message);
await circuitBreaker.Failure(exception).ConfigureAwait(false);
}
finally
{
// TODO: remove workaround when https://github.com/Azure/azure-service-bus-dotnet/issues/439 is fixed
Interlocked.Decrement(ref numberOfExecutingReceives);
}
// By default, ASB client long polls for a minute and returns null if it times out
if (message == null || messageProcessing.IsCancellationRequested)
{
return;
}
var lockToken = message.SystemProperties.LockToken;
string messageId;
Dictionary <string, string> headers;
byte[] body;
try
{
messageId = message.GetMessageId();
headers = message.GetNServiceBusHeaders();
body = message.GetBody();
}
catch (Exception exception)
{
try
{
await receiver.SafeDeadLetterAsync(pushSettings.RequiredTransactionMode, lockToken, deadLetterReason : "Poisoned message", deadLetterErrorDescription : exception.Message).ConfigureAwait(false);
}
catch (Exception)
{
// nothing we can do about it, message will be retried
}
return;
}
try
{
using (var receiveCancellationTokenSource = new CancellationTokenSource())
{
var transportTransaction = CreateTransportTransaction(message.PartitionKey);
var contextBag = new ContextBag();
contextBag.Set(message);
var messageContext = new MessageContext(messageId, headers, body, transportTransaction, receiveCancellationTokenSource, contextBag);
using (var scope = CreateTransactionScope())
{
await onMessage(messageContext).ConfigureAwait(false);
if (receiveCancellationTokenSource.IsCancellationRequested == false)
{
await receiver.SafeCompleteAsync(pushSettings.RequiredTransactionMode, lockToken).ConfigureAwait(false);
scope?.Complete();
}
}
if (receiveCancellationTokenSource.IsCancellationRequested)
{
await receiver.SafeAbandonAsync(pushSettings.RequiredTransactionMode, lockToken).ConfigureAwait(false);
}
}
}
catch (Exception exception)
{
try
{
ErrorHandleResult result;
using (var scope = CreateTransactionScope())
{
var transportTransaction = CreateTransportTransaction(message.PartitionKey);
var errorContext = new ErrorContext(exception, message.GetNServiceBusHeaders(), messageId, body, transportTransaction, message.SystemProperties.DeliveryCount);
result = await onError(errorContext).ConfigureAwait(false);
if (result == ErrorHandleResult.Handled)
{
await receiver.SafeCompleteAsync(pushSettings.RequiredTransactionMode, lockToken).ConfigureAwait(false);
}
scope?.Complete();
}
if (result == ErrorHandleResult.RetryRequired)
{
await receiver.SafeAbandonAsync(pushSettings.RequiredTransactionMode, lockToken).ConfigureAwait(false);
}
}
catch (Exception onErrorException) when(onErrorException is MessageLockLostException || onErrorException is ServiceBusTimeoutException)
{
logger.Debug("Failed to execute recoverability.", onErrorException);
}
catch (Exception onErrorException)
{
criticalError.Raise($"Failed to execute recoverability policy for message with native ID: `{message.MessageId}`", onErrorException);
await receiver.SafeAbandonAsync(pushSettings.RequiredTransactionMode, lockToken).ConfigureAwait(false);
}
}
}
async Task ReceiveMessage(CancellationToken messageReceivingCancellationToken)
{
ServiceBusReceivedMessage message = null;
try
{
message = await receiver.ReceiveMessageAsync(cancellationToken : messageReceivingCancellationToken).ConfigureAwait(false);
circuitBreaker.Success();
}
catch (ServiceBusException ex) when(ex.IsTransient)
{
}
catch (ObjectDisposedException)
{
// Can happen during endpoint shutdown
}
catch (Exception ex) when(!ex.IsCausedBy(messageReceivingCancellationToken))
{
Logger.Warn($"Failed to receive a message. Exception: {ex.Message}", ex);
await circuitBreaker.Failure(ex, messageReceivingCancellationToken).ConfigureAwait(false);
}
// By default, ASB client long polls for a minute and returns null if it times out
if (message == null)
{
return;
}
messageReceivingCancellationToken.ThrowIfCancellationRequested();
string messageId;
Dictionary <string, string> headers;
BinaryData body;
try
{
messageId = message.GetMessageId();
headers = message.GetNServiceBusHeaders();
body = message.GetBody();
}
catch (Exception ex)
{
var tryDeadlettering = transportSettings.TransportTransactionMode != TransportTransactionMode.None;
Logger.Warn($"Poison message detected. " +
$"Message {(tryDeadlettering ? "will be moved to the poison queue" : "will be discarded, transaction mode is set to None")}. " +
$"Exception: {ex.Message}", ex);
if (tryDeadlettering)
{
try
{
await receiver.DeadLetterMessageAsync(message, deadLetterReason : "Poisoned message", deadLetterErrorDescription : ex.Message, cancellationToken : messageReceivingCancellationToken).ConfigureAwait(false);
}
catch (Exception deadLetterEx) when(!deadLetterEx.IsCausedBy(messageReceivingCancellationToken))
{
// nothing we can do about it, message will be retried
Logger.Debug("Error dead lettering poisoned message.", deadLetterEx);
}
}
return;
}
// need to catch OCE here because we are switching token
try
{
await ProcessMessage(message, messageId, headers, body, messageProcessingCancellationTokenSource.Token).ConfigureAwait(false);
}
catch (Exception ex) when(ex.IsCausedBy(messageProcessingCancellationTokenSource.Token))
{
Logger.Debug("Message processing canceled.", ex);
}
}
