public Task Init(Func <MessageContext, Task> pump, Func <ErrorContext, Task <ErrorHandleResult> > onError, CriticalError criticalError, PushSettings pushSettings) { topologyOperator = DetermineTopologyOperator(pushSettings.InputQueue); messagePump = pump; var name = $"MessagePump on the queue `{pushSettings.InputQueue}`"; circuitBreaker = new RepeatedFailuresOverTimeCircuitBreaker(name, timeToWaitBeforeTriggering, ex => criticalError.Raise("Failed to receive message from Azure Service Bus.", ex)); if (pushSettings.PurgeOnStartup) { throw new InvalidOperationException("Azure Service Bus transport doesn't support PurgeOnStartup behavior"); } inputQueue = pushSettings.InputQueue; topologyOperator.OnIncomingMessage(async(incoming, receiveContext) => { if (circuitBreaker == null || throttler == null) /* can be disposed by fody injected logic, in theory */ { return; } var tokenSource = new CancellationTokenSource(); receiveContext.CancellationToken = tokenSource.Token; circuitBreaker.Success(); var transportTransaction = new TransportTransaction(); transportTransaction.Set(receiveContext); await throttler.WaitAsync(receiveContext.CancellationToken).ConfigureAwait(false); try { await messagePump(new MessageContext(incoming.MessageId, incoming.Headers, incoming.Body, transportTransaction, tokenSource, new ContextBag())).ConfigureAwait(false); } finally { throttler.Release(); } }); topologyOperator.OnError(exception => circuitBreaker?.Failure(exception)); topologyOperator.OnProcessingFailure(onError); topologyOperator.SetCriticalError(criticalError); return(TaskEx.Completed); }
async Task InnerProcessMessage(Task <Message> receiveTask) { Message message = null; try { // Workaround for ASB MessageReceiver.Receive() that has a timeout and doesn't take a CancellationToken. // We want to track how many receives are waiting and could be ignored when endpoint is stopping. // TODO: remove workaround when https://github.com/Azure/azure-service-bus-dotnet/issues/439 is fixed Interlocked.Increment(ref numberOfExecutingReceives); message = await receiveTask.ConfigureAwait(false); circuitBreaker.Success(); } catch (ServiceBusException sbe) when(sbe.IsTransient) { } catch (ObjectDisposedException) { // Can happen during endpoint shutdown } catch (Exception exception) { logger.WarnFormat("Failed to receive a message. Exception: {0}", exception.Message); await circuitBreaker.Failure(exception).ConfigureAwait(false); } finally { // TODO: remove workaround when https://github.com/Azure/azure-service-bus-dotnet/issues/439 is fixed Interlocked.Decrement(ref numberOfExecutingReceives); } // By default, ASB client long polls for a minute and returns null if it times out if (message == null || messageProcessing.IsCancellationRequested) { return; } var lockToken = message.SystemProperties.LockToken; string messageId; Dictionary <string, string> headers; byte[] body; try { messageId = message.GetMessageId(); headers = message.GetNServiceBusHeaders(); body = message.GetBody(); } catch (Exception exception) { try { await receiver.SafeDeadLetterAsync(pushSettings.RequiredTransactionMode, lockToken, deadLetterReason : "Poisoned message", deadLetterErrorDescription : exception.Message).ConfigureAwait(false); } catch (Exception) { // nothing we can do about it, message will be retried } return; } try { using (var receiveCancellationTokenSource = new CancellationTokenSource()) { var transportTransaction = CreateTransportTransaction(message.PartitionKey); var contextBag = new ContextBag(); contextBag.Set(message); var messageContext = new MessageContext(messageId, headers, body, transportTransaction, receiveCancellationTokenSource, contextBag); using (var scope = CreateTransactionScope()) { await onMessage(messageContext).ConfigureAwait(false); if (receiveCancellationTokenSource.IsCancellationRequested == false) { await receiver.SafeCompleteAsync(pushSettings.RequiredTransactionMode, lockToken).ConfigureAwait(false); scope?.Complete(); } } if (receiveCancellationTokenSource.IsCancellationRequested) { await receiver.SafeAbandonAsync(pushSettings.RequiredTransactionMode, lockToken).ConfigureAwait(false); } } } catch (Exception exception) { try { ErrorHandleResult result; using (var scope = CreateTransactionScope()) { var transportTransaction = CreateTransportTransaction(message.PartitionKey); var errorContext = new ErrorContext(exception, message.GetNServiceBusHeaders(), messageId, body, transportTransaction, message.SystemProperties.DeliveryCount); result = await onError(errorContext).ConfigureAwait(false); if (result == ErrorHandleResult.Handled) { await receiver.SafeCompleteAsync(pushSettings.RequiredTransactionMode, lockToken).ConfigureAwait(false); } scope?.Complete(); } if (result == ErrorHandleResult.RetryRequired) { await receiver.SafeAbandonAsync(pushSettings.RequiredTransactionMode, lockToken).ConfigureAwait(false); } } catch (Exception onErrorException) when(onErrorException is MessageLockLostException || onErrorException is ServiceBusTimeoutException) { logger.Debug("Failed to execute recoverability.", onErrorException); } catch (Exception onErrorException) { criticalError.Raise($"Failed to execute recoverability policy for message with native ID: `{message.MessageId}`", onErrorException); await receiver.SafeAbandonAsync(pushSettings.RequiredTransactionMode, lockToken).ConfigureAwait(false); } } }
async Task ReceiveMessage(CancellationToken messageReceivingCancellationToken) { ServiceBusReceivedMessage message = null; try { message = await receiver.ReceiveMessageAsync(cancellationToken : messageReceivingCancellationToken).ConfigureAwait(false); circuitBreaker.Success(); } catch (ServiceBusException ex) when(ex.IsTransient) { } catch (ObjectDisposedException) { // Can happen during endpoint shutdown } catch (Exception ex) when(!ex.IsCausedBy(messageReceivingCancellationToken)) { Logger.Warn($"Failed to receive a message. Exception: {ex.Message}", ex); await circuitBreaker.Failure(ex, messageReceivingCancellationToken).ConfigureAwait(false); } // By default, ASB client long polls for a minute and returns null if it times out if (message == null) { return; } messageReceivingCancellationToken.ThrowIfCancellationRequested(); string messageId; Dictionary <string, string> headers; BinaryData body; try { messageId = message.GetMessageId(); headers = message.GetNServiceBusHeaders(); body = message.GetBody(); } catch (Exception ex) { var tryDeadlettering = transportSettings.TransportTransactionMode != TransportTransactionMode.None; Logger.Warn($"Poison message detected. " + $"Message {(tryDeadlettering ? "will be moved to the poison queue" : "will be discarded, transaction mode is set to None")}. " + $"Exception: {ex.Message}", ex); if (tryDeadlettering) { try { await receiver.DeadLetterMessageAsync(message, deadLetterReason : "Poisoned message", deadLetterErrorDescription : ex.Message, cancellationToken : messageReceivingCancellationToken).ConfigureAwait(false); } catch (Exception deadLetterEx) when(!deadLetterEx.IsCausedBy(messageReceivingCancellationToken)) { // nothing we can do about it, message will be retried Logger.Debug("Error dead lettering poisoned message.", deadLetterEx); } } return; } // need to catch OCE here because we are switching token try { await ProcessMessage(message, messageId, headers, body, messageProcessingCancellationTokenSource.Token).ConfigureAwait(false); } catch (Exception ex) when(ex.IsCausedBy(messageProcessingCancellationTokenSource.Token)) { Logger.Debug("Message processing canceled.", ex); } }