protected async Task ProcessBrokeredMessage <TSubscriber>(Func <TSubscriber> subscriberFactory, BrokeredMessage brokeredMessage) where TSubscriber : IBusSubscriber
{
Type messageType = null;
var subscriberType = typeof(TSubscriber);
await _syncService.ExecuteAsync("WindowsServiceBus_" + subscriberType.Name, async() =>
{
try
{
messageType = GetMessageType(brokeredMessage.ContentType);
_logger.Debug(string.Format("{1}: processing '{0}' message ...", messageType.Name,
subscriberType.Name));
var message = GetBodyOfBase(brokeredMessage);
var subscriber = subscriberFactory();
var subscriberMethod = subscriber.GetType().GetMethod("Consume", new[] { messageType });
subscriberMethod.Invoke(subscriber, new object[] { message });
await brokeredMessage.CompleteAsync();
_logger.Debug(string.Format("{1}: processing '{0}' message finished.", message.GetType().Name,
subscriberType.Name));
}
catch (Exception ex)
{
_logger.Error(string.Format("{1}: error processing '{0}' message: {2}.",
messageType == null ? "unknown" : messageType.Name, subscriberType.Name, ex));
await brokeredMessage.AbandonAsync();
throw;
}
});
}
private async Task CreateOrderPaymentHandler(BrokeredMessage message)
{
try
{
var command = message.GetJsonBody <VerifyOrderPaymentCommand>();
if (command.Order.State == OrderState.Created &&
command.Order.Payment != null &&
command.Order.Payment.Status == PaymentStatus.Pending)
{
if (command.Order.Payment.ExpirationYear < DateTime.UtcNow.Year)
{
// failed
await OrderPaymentFailed(command.Order, "invalid payment year");
}
else
{
// succeeded
Random r = new Random();
await Task.Delay(1000 *r.Next(10));
await OrderSucceeded(command.Order);
}
}
}
catch (Exception ex)
{
await message.AbandonAsync();
}
}
public static void Run([ServiceBusTrigger("myqueue", AccessRights.Listen, Connection = "mandartestsb_SERVICEBUS")] BrokeredMessage message, TraceWriter log)
{
try
{
Stream stream = message.GetBody <Stream>();
StreamReader reader = new StreamReader(stream);
string s = reader.ReadToEnd();
log.Info($"message: {s}");
dynamic jsonObject = new JObject();
jsonObject.message = s;
if (s.Contains("3"))
{
log.Warning(" DeliveryCount:----> " + message.DeliveryCount.ToString());
message.AbandonAsync().Wait();
}
else
{
message.CompleteAsync().Wait();
}
client.CreateDocumentAsync(UriFactory.CreateDocumentCollectionUri(databaseName, collectionName), jsonObject).Wait();
}
catch (System.Exception ex)
{
throw ex;
}
}
private async Task processMessage(BrokeredMessage message, string logMessageStub)
{
try
{
// prepare to make callback request
HttpWebRequest newRequest = WebRequest.CreateHttp(callbackUrl);
newRequest.Method = "POST";
newRequest.ContentType = "application/json";
string postData = new ServiceBusMessage(message).body.ToString();
byte[] byteArray = Encoding.UTF8.GetBytes(postData);
// send to the callback Uri
Trace.WriteLine(String.Format("{0}: {1} Posting to callback: {2}",
logMessageStub, DateTime.Now.ToString("yyyyMMdd HHmmss"), callbackUrl));
using (Stream requestBody = newRequest.GetRequestStream())
{
requestBody.Write(byteArray, 0, byteArray.Length);
newRequest.GetResponse();
}
Trace.TraceInformation(String.Format("{0}: {1} Callback submitted successfully, workflow id: {2}",
logMessageStub, DateTime.Now.ToString("yyyyMMdd HHmmss"), workflowId));
await message.CompleteAsync();
}
catch (Exception ex)
{
Trace.TraceError(String.Format("{0}: {1} Callback failed, workflow id: {2}, {3}{4}",
logMessageStub, DateTime.Now.ToString("yyyyMMdd HHmmss"), workflowId, Environment.NewLine, ex.ToString()));
await message.AbandonAsync();
}
}
public override async Task CompleteProcessingMessageAsync(BrokeredMessage message, FunctionResult result, CancellationToken cancellationToken)
{
if (result.Succeeded)
{
if (!MessageOptions.AutoComplete)
{
// AutoComplete is true by default, but if set to false
// we need to complete the message
cancellationToken.ThrowIfCancellationRequested();
await message.CompleteAsync();
Console.WriteLine("Begin sleep");
//Sleep 5 seconds
Thread.Sleep(5000);
Console.WriteLine("Sleep 5 seconds");
}
}
else
{
cancellationToken.ThrowIfCancellationRequested();
await message.AbandonAsync();
}
}
/// <summary>
/// This method completes processing of the specified message, after the job function has been invoked.
/// </summary>
/// <param name="message">The message to complete processing for.</param>
/// <param name="result">The <see cref="FunctionResult"/> from the job invocation.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> to use</param>
/// <returns></returns>
public virtual async Task CompleteProcessingMessageAsync(BrokeredMessage message, FunctionResult result, CancellationToken cancellationToken)
{
if (!result.Succeeded)
{
cancellationToken.ThrowIfCancellationRequested();
await message.AbandonAsync();
}
}
private async Task ProcessMessageAsync(BrokeredMessage message, CancellationToken cancellationToken)
{
if (!await _triggerExecutor.ExecuteAsync(message, cancellationToken))
{
cancellationToken.ThrowIfCancellationRequested();
await message.AbandonAsync();
}
}
/// <summary>
/// This method completes processing of the specified message, after the job function has been invoked.
/// </summary>
/// <param name="message">The message to complete processing for.</param>
/// <param name="result">The <see cref="FunctionResult"/> from the job invocation.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> to use</param>
/// <returns></returns>
public virtual async Task CompleteProcessingMessageAsync(BrokeredMessage message, FunctionResult result, CancellationToken cancellationToken)
{
if (!result.Succeeded)
{
cancellationToken.ThrowIfCancellationRequested();
await message.AbandonAsync();
}
}
private async Task ProcessMessageAsync(BrokeredMessage message, CancellationToken cancellationToken)
{
FunctionResult result = await _triggerExecutor.ExecuteAsync(message, cancellationToken);
if (!result.Succeeded)
{
cancellationToken.ThrowIfCancellationRequested();
await message.AbandonAsync();
}
}
public static async Task <bool> SafeAbandonAsync(this BrokeredMessage msg, IDictionary <string, object> propertiesToModify = null)
{
try
{
if (propertiesToModify == null)
{
await msg.AbandonAsync().ConfigureAwait(false);
}
else
{
await msg.AbandonAsync(propertiesToModify).ConfigureAwait(false);
}
return(true);
}
catch (MessageLockLostException ex)
{
// It's too late to compensate the loss of a message lock. We should just ignore it so that it does not break the receive loop.
Log.Warn($"A message lock lost exception occurred while trying to abandon a message, you may consider to increase the lock duration or reduce the prefetch count, the exception was {ex.Message}", ex);
}
catch (MessagingException ex)
{
// There is nothing we can do as the connection may have been lost, or the underlying queue may have been removed.
// If Abandon() fails with this exception, the only recourse is to receive another message.
Log.Warn($"A messaging exception occurred while trying to abandon a message, this might imply that the connection was lost or the underlying queue got removed, the exception was {ex.Message}", ex);
}
catch (ObjectDisposedException ex)
{
// There is nothing we can do as the object has already been disposed elsewhere
Log.Warn($"An object disposed exception occurred while trying to abandon a message, this might imply that the connection was lost or the underlying queue got removed, the exception was {ex.Message}", ex);
}
catch (TransactionException ex)
{
// ASB Sdk beat us to it
Log.Warn($"A transaction exception occurred while trying to abandon a message, this probably means that the Azure ServiceBus SDK has rolled back the transaction already, the exception was {ex.Message}", ex);
}
catch (TimeoutException ex)
{
// took to long
Log.Warn($"A timeout exception occurred while trying to abandon a message, the exception was {ex.Message}", ex);
}
return(false);
}
async Task IMessageSessionAsyncHandler.OnMessageAsync(MessageSession session, BrokeredMessage message)
{
if (_receiver.IsShuttingDown)
{
await WaitAndAbandonMessage(message).ConfigureAwait(false);
return;
}
using (var delivery = _tracker.BeginDelivery())
{
if (_log.IsDebugEnabled)
{
_log.DebugFormat("Receiving {0}:{1}({3}) - {2}", delivery.Id, message.MessageId, _receiver.QueuePath, session.SessionId);
}
var context = new ServiceBusReceiveContext(_receiver.InputAddress, message, _context, _sendEndpointProvider, _publishEndpointProvider);
context.GetOrAddPayload <MessageSessionContext>(() => new BrokeredMessageSessionContext(session));
context.GetOrAddPayload(() => _context);
try
{
await _context.PreReceive(context).ConfigureAwait(false);
await _receiver.ReceivePipe.Send(context).ConfigureAwait(false);
await context.CompleteTask.ConfigureAwait(false);
await message.CompleteAsync().ConfigureAwait(false);
await _context.PostReceive(context).ConfigureAwait(false);
if (_log.IsDebugEnabled)
{
_log.DebugFormat("Receive completed: {0}", message.MessageId);
}
}
catch (Exception ex)
{
if (_log.IsErrorEnabled)
{
_log.Error($"Received faulted: {message.MessageId}", ex);
}
await message.AbandonAsync().ConfigureAwait(false);
await _context.ReceiveFault(context, ex).ConfigureAwait(false);
}
finally
{
context.Dispose();
}
}
}
async Task IMessageSessionAsyncHandler.OnMessageAsync(MessageSession session, BrokeredMessage message)
{
if (_receiver.IsShuttingDown)
{
await WaitAndAbandonMessage(message).ConfigureAwait(false);
return;
}
var deliveryCount = _receiver.IncrementDeliveryCount();
if (_log.IsDebugEnabled)
{
_log.DebugFormat("Receiving {0}:{1}({3}) - {2}", deliveryCount, message.MessageId, _receiver.QueuePath, session.SessionId);
}
var context = new ServiceBusReceiveContext(_receiver.InputAddress, message, _receiver.ReceiveObserver);
context.GetOrAddPayload <MessageSessionContext>(() => new BrokeredMessageSessionContext(session));
try
{
await _receiver.ReceiveObserver.PreReceive(context).ConfigureAwait(false);
await _receiver.ReceivePipe.Send(context).ConfigureAwait(false);
await context.CompleteTask.ConfigureAwait(false);
await message.CompleteAsync().ConfigureAwait(false);
await _receiver.ReceiveObserver.PostReceive(context).ConfigureAwait(false);
if (_log.IsDebugEnabled)
{
_log.DebugFormat("Receive completed: {0}", message.MessageId);
}
}
catch (Exception ex)
{
if (_log.IsErrorEnabled)
{
_log.Error($"Received faulted: {message.MessageId}", ex);
}
await message.AbandonAsync().ConfigureAwait(false);
await _receiver.ReceiveObserver.ReceiveFault(context, ex).ConfigureAwait(false);
}
finally
{
_receiver.DeliveryComplete();
}
}
async Task WaitAndAbandonMessage(BrokeredMessage message)
{
try
{
await _deliveryComplete.Task.ConfigureAwait(false);
await message.AbandonAsync().ConfigureAwait(false);
}
catch (Exception exception)
{
LogContext.Error?.Log(exception, "Abandon message faulted during shutdown: {InputAddress}", _context.InputAddress);
}
}
private async Task HandleDispatchCompletion(Task t, BrokeredMessage m)
{
if (t.IsFaulted)
{
var exception = t.Exception;
Logger.Error(exception, "Message dispatch failed");
await m.AbandonAsync(ExceptionDetailsAsProperties(exception));
return;
}
Logger.Debug("Dispatched message: {0} from {1}", m, m.ReplyTo);
await m.CompleteAsync();
}
public static void SafeAbandonAsync(this BrokeredMessage message, string subscription, Action <bool> callback, long processingElapsedMilliseconds, long schedulingElapsedMilliseconds, Stopwatch roundtripStopwatch)
{
SafeMessagingActionAsync(
message.AbandonAsync(),
message,
callback,
"An error occurred while abandoning message {0} in subscription {1} with processing time {3} (scheduling {4} request {5} roundtrip {6}). Error message: {2}",
message.MessageId,
subscription,
processingElapsedMilliseconds,
schedulingElapsedMilliseconds,
roundtripStopwatch);
}
static async Task AbandonMessage(BrokeredMessage message)
{
try
{
await message.AbandonAsync().ConfigureAwait(false);
}
catch (Exception exception)
{
if (_log.IsWarnEnabled)
{
_log.Warn($"Abandon message faulted: {message.MessageId}", exception);
}
}
}
async Task WaitAndAbandonMessage(BrokeredMessage message)
{
try
{
await _deliveryComplete.Task.ConfigureAwait(false);
await message.AbandonAsync().ConfigureAwait(false);
}
catch (Exception exception)
{
if (_log.IsErrorEnabled)
_log.Debug($"Stopping receiver, abandoned message faulted: {_context.InputAddress}", exception);
}
}
async Task IBrokeredMessageReceiver.Handle(BrokeredMessage message, Action <ReceiveContext> contextCallback)
{
var context = new ServiceBusReceiveContext(_inputAddress, message, _receiveObservers, _receiveTopology);
contextCallback?.Invoke(context);
try
{
await _receiveObservers.PreReceive(context).ConfigureAwait(false);
if (message.LockedUntilUtc <= DateTime.UtcNow)
{
throw new MessageLockExpiredException(_inputAddress, $"The message lock expired: {message.MessageId}");
}
if (message.ExpiresAtUtc < DateTime.UtcNow)
{
throw new MessageTimeToLiveExpiredException(_inputAddress, $"The message TTL expired: {message.MessageId}");
}
await _receivePipe.Send(context).ConfigureAwait(false);
await context.CompleteTask.ConfigureAwait(false);
await message.CompleteAsync().ConfigureAwait(false);
await _receiveObservers.PostReceive(context).ConfigureAwait(false);
}
catch (Exception ex)
{
try
{
await message.AbandonAsync().ConfigureAwait(false);
}
catch (Exception exception)
{
if (_log.IsWarnEnabled)
{
_log.Warn($"Abandon message faulted: {message.MessageId}", exception);
}
}
await _receiveObservers.ReceiveFault(context, ex).ConfigureAwait(false);
}
finally
{
context.Dispose();
}
}
async Task WaitAndAbandonMessage(BrokeredMessage message)
{
try
{
await _receiver.DeliveryCompleted.ConfigureAwait(false);
await message.AbandonAsync().ConfigureAwait(false);
}
catch (Exception exception)
{
if (_log.IsErrorEnabled)
{
_log.Debug("Stopping async handler, abandoned message faulted: {_inputAddress}", exception);
}
}
}
async Task WaitAndAbandonMessage(BrokeredMessage message)
{
try
{
await _participant.ParticipantCompleted.ConfigureAwait(false);
await message.AbandonAsync().ConfigureAwait(false);
}
catch (Exception exception)
{
if (_log.IsErrorEnabled)
{
_log.Debug("Shutting down, abandoned message faulted: {_clientContext.InputAddress}", exception);
}
}
}
private async Task MessageCleanupAsync(BrokeredMessage message, FunctionResult result, CancellationToken cancellationToken)
{
if (result.Succeeded)
{
if (!_options.AutoComplete)
{
cancellationToken.ThrowIfCancellationRequested();
await message.CompleteAsync();
}
}
else
{
cancellationToken.ThrowIfCancellationRequested();
await message.AbandonAsync();
}
}
private async Task EventReceived(BrokeredMessage m)
{
Trace.WriteLine("Message Event Received");
var shouldAbandon = false;
try
{
// TODO: Use reflection instead
switch (m.ContentType)
{
case "Models.ProtoBuf.Event":
{
// Deserialize Message
var msg = m.GetBody <Event>(_xpsEvent);
await ProcessEvent(msg); //.ConfigureAwait(false);
break;
}
case "Models.ProtoBuf.Command":
{
// Deserialize Message
var msg = m.GetBody <Command>(_xpsCommand);
await ProcessCommand(msg); //.ConfigureAwait(false);
break;
}
}
await m.CompleteAsync();//.ConfigureAwait(false);
}
catch
{
Trace.TraceError("Message Event Threw");
// NOTE: Can't await inside a catch... yet
shouldAbandon = true;
// TODO: Handle message processing specific exceptions
}
if (shouldAbandon)
{
await m.AbandonAsync();//.ConfigureAwait(false);
}
}
async Task IMessageSessionAsyncHandler.OnMessageAsync(MessageSession session, BrokeredMessage message)
{
if (_receiver.IsShuttingDown)
{
await WaitAndAbandonMessage(message).ConfigureAwait(false);
return;
}
using (var delivery = _tracker.BeginDelivery())
{
if (_log.IsDebugEnabled)
_log.DebugFormat("Receiving {0}:{1}({3}) - {2}", delivery.Id, message.MessageId, _receiver.QueuePath, session.SessionId);
var context = new ServiceBusReceiveContext(_receiver.InputAddress, message, _context, _sendEndpointProvider, _publishEndpointProvider);
context.GetOrAddPayload<MessageSessionContext>(() => new BrokeredMessageSessionContext(session));
context.GetOrAddPayload(() => _context);
try
{
await _context.PreReceive(context).ConfigureAwait(false);
await _receiver.ReceivePipe.Send(context).ConfigureAwait(false);
await context.CompleteTask.ConfigureAwait(false);
await message.CompleteAsync().ConfigureAwait(false);
await _context.PostReceive(context).ConfigureAwait(false);
if (_log.IsDebugEnabled)
_log.DebugFormat("Receive completed: {0}", message.MessageId);
}
catch (Exception ex)
{
if (_log.IsErrorEnabled)
_log.Error($"Received faulted: {message.MessageId}", ex);
await message.AbandonAsync().ConfigureAwait(false);
await _context.ReceiveFault(context, ex).ConfigureAwait(false);
}
finally
{
context.Dispose();
}
}
}
/// <summary>
/// This method completes processing of the specified message, after the job function has been invoked.
/// </summary>
/// <param name="message">The message to complete processing for.</param>
/// <param name="result">The <see cref="FunctionResult"/> from the job invocation.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> to use</param>
/// <returns>A <see cref="Task"/> that will complete the message processing.</returns>
public virtual async Task CompleteProcessingMessageAsync(BrokeredMessage message, FunctionResult result, CancellationToken cancellationToken)
{
if (result.Succeeded)
{
if (!MessageOptions.AutoComplete)
{
// AutoComplete is true by default, but if set to false
// we need to complete the message
cancellationToken.ThrowIfCancellationRequested();
await message.CompleteAsync();
}
}
else
{
cancellationToken.ThrowIfCancellationRequested();
await message.AbandonAsync();
}
}
/// <summary>
/// This method completes processing of the specified message, after the job function has been invoked.
/// </summary>
/// <param name="message">The message to complete processing for.</param>
/// <param name="result">The <see cref="FunctionResult"/> from the job invocation.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> to use</param>
/// <returns>A <see cref="Task"/> that will complete the message processing.</returns>
public virtual async Task CompleteProcessingMessageAsync(BrokeredMessage message, FunctionResult result, CancellationToken cancellationToken)
{
if (result.Succeeded)
{
if (!MessageOptions.AutoComplete)
{
// AutoComplete is true by default, but if set to false
// we need to complete the message
cancellationToken.ThrowIfCancellationRequested();
await message.CompleteAsync();
}
}
else
{
cancellationToken.ThrowIfCancellationRequested();
await message.AbandonAsync();
}
}
async Task IMessageSessionAsyncHandler.OnMessageAsync(MessageSession session, BrokeredMessage message)
{
if (_receiver.IsShuttingDown)
{
await WaitAndAbandonMessage(message).ConfigureAwait(false);
return;
}
var deliveryCount = _receiver.IncrementDeliveryCount();
if (_log.IsDebugEnabled)
_log.DebugFormat("Receiving {0}:{1}({3}) - {2}", deliveryCount, message.MessageId, _receiver.QueuePath, session.SessionId);
var context = new ServiceBusReceiveContext(_receiver.InputAddress, message, _receiver.ReceiveObserver);
context.GetOrAddPayload<MessageSessionContext>(() => new BrokeredMessageSessionContext(session));
try
{
await _receiver.ReceiveObserver.PreReceive(context).ConfigureAwait(false);
await _receiver.ReceivePipe.Send(context).ConfigureAwait(false);
await context.CompleteTask.ConfigureAwait(false);
await message.CompleteAsync().ConfigureAwait(false);
await _receiver.ReceiveObserver.PostReceive(context).ConfigureAwait(false);
if (_log.IsDebugEnabled)
_log.DebugFormat("Receive completed: {0}", message.MessageId);
}
catch (Exception ex)
{
if (_log.IsErrorEnabled)
_log.Error($"Received faulted: {message.MessageId}", ex);
await message.AbandonAsync().ConfigureAwait(false);
await _receiver.ReceiveObserver.ReceiveFault(context, ex).ConfigureAwait(false);
}
finally
{
_receiver.DeliveryComplete();
}
}
async Task Receive(BrokeredMessage message)
{
Log.InfoFormat("收到ID为{0}的事件", message.MessageId);
if (message.Properties.ContainsKey(Config.TypeNameProperty))
{
string typeName = message.Properties[Config.TypeNameProperty].ToString();
string messageBody = message.GetBody <string>();
if (EventDispatch.Dispatch(typeName, messageBody))
{
await message.CompleteAsync();
Log.InfoFormat("事件ID为{0}的事件处理完毕", message.MessageId);
return;
}
}
await message.AbandonAsync();
Log.InfoFormat("事件ID为{0}的事件处理失败", message.MessageId);
}
async Task OnMessage(BrokeredMessage brokeredMessage)
{
Contract.Requires(brokeredMessage != null);
var message = FromBrokeredMessage(brokeredMessage);
var currentObservers = default(IEnumerable <IObserver <IMessageDescriptor> >);
lock ( syncRoot )
{
currentObservers = observers.ToArray();
}
try
{
foreach (var observer in currentObservers)
{
observer.OnNext(message);
}
}
catch (TimeoutException error)
{
foreach (var observer in currentObservers)
{
observer.OnError(error);
}
}
catch (OperationCanceledException error)
{
foreach (var observer in currentObservers)
{
observer.OnError(error);
}
}
catch
{
await brokeredMessage.AbandonAsync().ConfigureAwait(false);
return;
}
await brokeredMessage.CompleteAsync().ConfigureAwait(false);
}
/// <summary>
/// Handles processing messages from a NON Session based queue
/// </summary>
/// <param name="message">The broker message from the Service Bus</param>
private static async Task HandleMessage(BrokeredMessage message)
{
try
{
Utility.ParseExecutionContextFromBrokeredMessage(message);
// This will remove the message from the topic
await message.CompleteAsync();
}
catch (Exception ex)
{
if (!(ex is MessageLockLostException))
{
//Let service bus know we failed to process the message
await message.AbandonAsync();
throw;
}
}
}
private void ReleaseMessage(BrokeredMessage msg, MessageReleaseAction releaseAction, long processingElapsedMilliseconds, long schedulingElapsedMilliseconds, Stopwatch roundtripStopwatch)
{
switch (releaseAction.Kind)
{
case MessageReleaseActionKind.Complete:
msg.CompleteAsync();
break;
case MessageReleaseActionKind.Abandon:
msg.AbandonAsync();
break;
case MessageReleaseActionKind.DeadLetter:
msg.DeadLetterAsync();
break;
default:
break;
}
}
public async Task SafeAbandonAsync(BrokeredMessage message)
{
try
{
if (_receiveMode == ReceiveMode.ReceiveAndDelete)
{
Log.Warning(
"Not expected to abandon messages on the subscription \"{0}\" that has ReceiveMode={1}.",
this, _receiveMode);
return;
}
await RetryPolicy.ExecuteAsync(async() => await message.AbandonAsync());
}
// ReSharper disable once EmptyGeneralCatchClause
catch (Exception)
{
// It does not matter if we fail to abandon.
// Most probably, that only means that it already has been abandoned.
}
}
public async Task AbandonAsync()
{
using (await _asyncLock.LockAsync())
{
_timer.Dispose();
if (_message == null)
{
return;
}
try
{
await _message.AbandonAsync();
}
catch /*(Exception e)*/
{
// log it
}
_message = null;
}
}
private async Task Dispatch(BrokeredMessage message)
{
// Early exit: have we pre-fetched this message and had our lock already expire? If so, just
// bail - it will already have been picked up by someone else.
if (message.LockedUntilUtc <= _clock.UtcNow)
{
GlobalMessageCounters.IncrementMessagesDroppedDueToExpiredLock(1);
_logger.Debug("Lock for message {MessageId} appears to have already expired so we're not dispatching it. Watch out for clock drift between your service bus server and {MachineName}!", message.MessageId, Environment.MachineName);
return;
}
try
{
Exception exception = null;
try
{
LogInfo("Dispatching", message);
using (_dispatchContextManager.StartNewDispatchContext(new SubsequentDispatchContext(message)))
{
await _taskFactory.StartNew(() => _messageDispatcher.Dispatch(message), TaskContext.Dispatch).Unwrap();
}
LogDebug("Dispatched", message);
LogDebug("Completing", message);
message.Properties[MessagePropertyKeys.DispatchComplete] = true;
await _taskFactory.StartNew(() => message.CompleteAsync(), TaskContext.CompleteOrAbandon).Unwrap();
LogInfo("Completed", message);
return;
}
catch (Exception exc)
{
if (exc is MessageLockLostException || (exc.InnerException is MessageLockLostException))
{
_logger.Error(exc,
"Message completion failed for {Type} from {QueuePath} [MessageId:{MessageId}, CorrelationId:{CorrelationId}]",
message.SafelyGetBodyTypeNameOrDefault(),
message.ReplyTo,
message.MessageId,
message.CorrelationId);
return;
}
_logger.Error(exc,
"Message dispatch failed for {Type} from {QueuePath} [MessageId:{MessageId}, CorrelationId:{CorrelationId}]",
message.SafelyGetBodyTypeNameOrDefault(),
message.ReplyTo,
message.MessageId,
message.CorrelationId);
exception = exc;
}
try
{
LogDebug("Abandoning", message);
message.Properties[MessagePropertyKeys.DispatchAbandoned] = true;
await message.AbandonAsync(exception.ExceptionDetailsAsProperties(_clock.UtcNow));
LogDebug("Abandoned", message);
}
catch (Exception exc)
{
_logger.Error(exc,
"Could not call Abandon() on message {Type} from {QueuePath} [MessageId:{MessageId}, CorrelationId:{CorrelationId}].",
message.SafelyGetBodyTypeNameOrDefault(),
message.MessageId,
message.CorrelationId,
message.ReplyTo);
}
}
catch (Exception exc)
{
_logger.Fatal(exc, "Unhandled exception in message pump");
}
}
internal async Task Process(BrokeredMessage brokeredMessage)
{
try
{
var stream = brokeredMessage.GetBody <Stream>();
var message = _brokeredMessageFactory.From(stream);
var handlerResult = await _handlerFactory.Handle(message);
var attemptedTimestamp = DateTimeProvider.Instance.Now;
var publishedAt = new DateTimeOffset(brokeredMessage.EnqueuedTimeUtc);
var durationSincePublishMs = DateTimeProvider.Instance.Now.ToUnixTimeMilliseconds() - publishedAt.ToUnixTimeMilliseconds();
switch (handlerResult.Status)
{
case HandlerResult.StatusCodes.HandledSuccessfully:
await brokeredMessage.CompleteAsync();
Logging.Logger
.ForContext(message.GetType())
.ForContext("MessageId", brokeredMessage.MessageId)
.ForContext("CorrelationId", brokeredMessage.CorrelationId)
.ForContext("Attempt", brokeredMessage.DeliveryCount)
.ForContext("PublishedAt", publishedAt)
.ForContext("AttemptedAt", attemptedTimestamp)
.ForContext("HandleDurationMs", handlerResult.DurationMs)
.ForContext("DurationSincePublishMs", durationSincePublishMs)
//.ForContext("FromQueue", _queue.QueueName)
.Information("Message handled.");
break;
case HandlerResult.StatusCodes.MessageHandlerNotFound:
await brokeredMessage.AbandonAsync();
Logging.Logger
.ForContext(message.GetType())
.ForContext("MessageId", brokeredMessage.MessageId)
.ForContext("CorrelationId", brokeredMessage.CorrelationId)
.ForContext("Attempt", brokeredMessage.DeliveryCount)
.ForContext("PublishedAt", publishedAt)
.ForContext("AttemptedAt", attemptedTimestamp)
.ForContext("DurationSincePublishMs", durationSincePublishMs)
//.ForContext("FromQueue", _queue.QueueName)
.Error("Message handler not found for message type.");
break;
case HandlerResult.StatusCodes.UnhandledErrorDuringHandlerExecution:
await brokeredMessage.AbandonAsync();
Logging.Logger
.ForContext(message.GetType())
.ForContext("MessageId", brokeredMessage.MessageId)
.ForContext("CorrelationId", brokeredMessage.CorrelationId)
.ForContext("Attempt", brokeredMessage.DeliveryCount)
.ForContext("PublishedAt", publishedAt)
.ForContext("AttemptedAt", attemptedTimestamp)
.ForContext("DurationSincePublishMs", durationSincePublishMs)
//.ForContext("FromQueue", _queue.QueueName)
.Error(handlerResult.Exception, "Unhandled error occured while executing message handler.");
break;
default:
throw new ArgumentOutOfRangeException();
}
}
catch (Exception exception)
{
await brokeredMessage.AbandonAsync();
var stream = brokeredMessage.GetBody <Stream>();
var body = new StreamReader(stream, true).ReadToEnd();
Logging.Logger
.ForContext("MessageId", brokeredMessage.MessageId)
.ForContext("CorrelationId", brokeredMessage.CorrelationId)
.ForContext("Body", body)
//.ForContext("FromQueue", _queue.QueueName)
.Error(exception, "Unhandled error occured consuming message.");
}
}
private async Task EventReceived(BrokeredMessage m)
{
Trace.WriteLine("Message Event Received");
var shouldAbandon = false;
try
{
// TODO: Use reflection instead
switch (m.ContentType)
{
case "Models.ProtoBuf.Event":
{
// Deserialize Message
var msg = m.GetBody<Event>(_xpsEvent);
await ProcessEvent(msg);//.ConfigureAwait(false);
break;
}
case "Models.ProtoBuf.Command":
{
// Deserialize Message
var msg = m.GetBody<Command>(_xpsCommand);
await ProcessCommand(msg);//.ConfigureAwait(false);
break;
}
}
await m.CompleteAsync();//.ConfigureAwait(false);
}
catch
{
Trace.TraceError("Message Event Threw");
// NOTE: Can't await inside a catch... yet
shouldAbandon = true;
// TODO: Handle message processing specific exceptions
}
if (shouldAbandon)
{
await m.AbandonAsync();//.ConfigureAwait(false);
}
}
private async Task ProcessMessageAsync(BrokeredMessage message, CancellationToken cancellationToken)
{
FunctionResult result = await _triggerExecutor.ExecuteAsync(message, cancellationToken);
if (!result.Succeeded)
{
cancellationToken.ThrowIfCancellationRequested();
await message.AbandonAsync();
}
}
public static async Task CancelRentalCar(BrokeredMessage message, MessageSender nextStepQueue, MessageSender compensatorQueue)
{
try
{
var via = (message.Properties.ContainsKey("Via")
? ((string)message.Properties["Via"] + ",")
: string.Empty) +
"cancelcar";
using (var scope = new TransactionScope(TransactionScopeAsyncFlowOption.Enabled))
{
if (message.Label != null &&
message.ContentType != null &&
message.Label.Equals(TravelBookingLabel, StringComparison.InvariantCultureIgnoreCase) &&
message.ContentType.Equals(ContentTypeApplicationJson, StringComparison.InvariantCultureIgnoreCase))
{
var body = message.GetBody <Stream>();
dynamic travelBooking = DeserializeTravelBooking(body);
// do we want to book a flight? No? Let's just forward the message to
// the next destination via transfer queue
if (travelBooking.car != null &&
travelBooking.car.reservationId != null)
{
lock (Console.Out)
{
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine("Cancelling Rental Car");
Console.ResetColor();
}
// undo the reservation (or pretend to fail)
if (DateTime.UtcNow.Second <= 3)
{
throw new Exception("O_o");
}
// reset the id
travelBooking.car.reservationId = null;
// forward
await nextStepQueue.SendAsync(CreateForwardMessage(message, travelBooking, via));
}
else
{
await nextStepQueue.SendAsync(CreateForwardMessage(message, travelBooking, via));
}
// done with this job
await message.CompleteAsync();
}
else
{
await message.DeadLetterAsync(
new Dictionary <string, object>
{
{ "DeadLetterReason", "BadMessage" },
{ "DeadLetterErrorDescription", "Unrecognized input message" },
{ "Via", via }
});
}
scope.Complete();
}
}
catch (Exception e)
{
Trace.TraceError(e.ToString());
await message.AbandonAsync();
}
}
public static async Task BookFlight(BrokeredMessage message, MessageSender nextStepQueue, MessageSender compensatorQueue)
{
try
{
using (var scope = new TransactionScope(TransactionScopeAsyncFlowOption.Enabled))
{
var via = (message.Properties.ContainsKey("Via")
? ((string)message.Properties["Via"] + ",")
: string.Empty) +
"bookflight";
if (message.Label != null &&
message.ContentType != null &&
message.Label.Equals(TravelBookingLabel, StringComparison.InvariantCultureIgnoreCase) &&
message.ContentType.Equals(ContentTypeApplicationJson, StringComparison.InvariantCultureIgnoreCase))
{
var body = message.GetBody <Stream>();
dynamic travelBooking = DeserializeTravelBooking(body);
// do we want to book a flight? No? Let's just forward the message to
// the next destination via transfer queue
if (travelBooking.flight == null)
{
await nextStepQueue.SendAsync(CreateForwardMessage(message, travelBooking, via));
// done with this job
await message.CompleteAsync();
}
else
{
lock (Console.Out)
{
Console.ForegroundColor = ConsoleColor.Cyan;
Console.WriteLine("Booking Flight");
Console.ResetColor();
}
// now we're going to simulate the work of booking a flight,
// which usually involves a call to a third party
// every 9th flight booking sadly goes wrong
if (message.SequenceNumber % 9 == 0)
{
await message.DeadLetterAsync(
new Dictionary <string, object>
{
{ "DeadLetterReason", "TransactionError" },
{ "DeadLetterErrorDescription", "Failed to perform flight reservation" },
{ "Via", via }
});
}
else
{
// every operation executed in the first 3 secs of any minute
// tanks completely (simulates some local or external unexpected issue)
if (DateTime.UtcNow.Second <= 3)
{
throw new Exception("O_o");
}
// let's pretend we booked something
travelBooking.flight.reservationId = "A1B2C3";
await nextStepQueue.SendAsync(CreateForwardMessage(message, travelBooking, via));
// done with this job
await message.CompleteAsync();
}
}
}
else
{
await message.DeadLetterAsync(
new Dictionary <string, object>
{
{ "DeadLetterReason", "BadMessage" },
{ "DeadLetterErrorDescription", "Unrecognized input message" },
{ "Via", via }
});
}
scope.Complete();
}
}
catch (Exception e)
{
Trace.TraceError(e.ToString());
await message.AbandonAsync();
}
}
public static async Task BookFlight(BrokeredMessage message, MessageSender nextStepQueue, MessageSender compensatorQueue)
{
try
{
using (var scope = new TransactionScope(TransactionScopeAsyncFlowOption.Enabled))
{
var via = (message.Properties.ContainsKey("Via")
? ((string) message.Properties["Via"] + ",")
: string.Empty) +
"bookflight";
if (message.Label != null &&
message.ContentType != null &&
message.Label.Equals(TravelBookingLabel, StringComparison.InvariantCultureIgnoreCase) &&
message.ContentType.Equals(ContentTypeApplicationJson, StringComparison.InvariantCultureIgnoreCase))
{
var body = message.GetBody<Stream>();
dynamic travelBooking = DeserializeTravelBooking(body);
// do we want to book a flight? No? Let's just forward the message to
// the next destination via transfer queue
if (travelBooking.flight == null)
{
await nextStepQueue.SendAsync(CreateForwardMessage(message, travelBooking, via));
// done with this job
await message.CompleteAsync();
}
else
{
lock (Console.Out)
{
Console.ForegroundColor = ConsoleColor.Cyan;
Console.WriteLine("Booking Flight");
Console.ResetColor();
}
// now we're going to simulate the work of booking a flight,
// which usually involves a call to a third party
// every 9th flight booking sadly goes wrong
if (message.SequenceNumber%9 == 0)
{
await message.DeadLetterAsync(
new Dictionary<string, object>
{
{"DeadLetterReason", "TransactionError"},
{"DeadLetterErrorDescription", "Failed to perform flight reservation"},
{"Via", via}
});
}
else
{
// every operation executed in the first 3 secs of any minute
// tanks completely (simulates some local or external unexpected issue)
if (DateTime.UtcNow.Second <= 3)
{
throw new Exception("O_o");
}
// let's pretend we booked something
travelBooking.flight.reservationId = "A1B2C3";
await nextStepQueue.SendAsync(CreateForwardMessage(message, travelBooking, via));
// done with this job
await message.CompleteAsync();
}
}
}
else
{
await message.DeadLetterAsync(
new Dictionary<string, object>
{
{"DeadLetterReason", "BadMessage"},
{"DeadLetterErrorDescription", "Unrecognized input message"},
{"Via", via}
});
}
scope.Complete();
}
}
catch (Exception e)
{
Trace.TraceError(e.ToString());
await message.AbandonAsync();
}
}
public static async Task CancelRentalCar(BrokeredMessage message, MessageSender nextStepQueue, MessageSender compensatorQueue)
{
try
{
var via = (message.Properties.ContainsKey("Via")
? ((string) message.Properties["Via"] + ",")
: string.Empty) +
"cancelcar";
using (var scope = new TransactionScope(TransactionScopeAsyncFlowOption.Enabled))
{
if (message.Label != null &&
message.ContentType != null &&
message.Label.Equals(TravelBookingLabel, StringComparison.InvariantCultureIgnoreCase) &&
message.ContentType.Equals(ContentTypeApplicationJson, StringComparison.InvariantCultureIgnoreCase))
{
var body = message.GetBody<Stream>();
dynamic travelBooking = DeserializeTravelBooking(body);
// do we want to book a flight? No? Let's just forward the message to
// the next destination via transfer queue
if (travelBooking.car != null &&
travelBooking.car.reservationId != null)
{
lock (Console.Out)
{
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine("Cancelling Rental Car");
Console.ResetColor();
}
// undo the reservation (or pretend to fail)
if (DateTime.UtcNow.Second <= 3)
{
throw new Exception("O_o");
}
// reset the id
travelBooking.car.reservationId = null;
// forward
await nextStepQueue.SendAsync(CreateForwardMessage(message, travelBooking, via));
}
else
{
await nextStepQueue.SendAsync(CreateForwardMessage(message, travelBooking, via));
}
// done with this job
await message.CompleteAsync();
}
else
{
await message.DeadLetterAsync(
new Dictionary<string, object>
{
{"DeadLetterReason", "BadMessage"},
{"DeadLetterErrorDescription", "Unrecognized input message"},
{"Via", via}
});
}
scope.Complete();
}
}
catch (Exception e)
{
Trace.TraceError(e.ToString());
await message.AbandonAsync();
}
}
private async Task Dispatch(BrokeredMessage message)
{
try
{
Exception exception = null;
try
{
LogInfo("Dispatching", message);
using (_dispatchContextManager.StartNewDispatchContext(new DispatchContext(message)))
{
await _messageDispatcher.Dispatch(message);
}
LogDebug("Dispatched", message);
LogDebug("Completing", message);
await message.CompleteAsync();
LogInfo("Completed", message);
return;
}
catch (Exception exc)
{
exception = exc;
}
_logger.Error(exception,
"Message dispatch failed for {0} from {1} [MessageId:{2}, CorrelationId:{3}]",
message.SafelyGetBodyTypeNameOrDefault(),
message.ReplyTo,
message.MessageId,
message.CorrelationId);
try
{
LogDebug("Abandoning", message);
await message.AbandonAsync(exception.ExceptionDetailsAsProperties(_clock.UtcNow));
LogDebug("Abandoned", message);
}
catch (Exception exc)
{
_logger.Error(exc,
"Could not call Abandon() on message {0} from {1} [MessageId:{2}, CorrelationId:{3}]. Possible lock expiry?",
message.SafelyGetBodyTypeNameOrDefault(),
message.MessageId,
message.CorrelationId,
message.ReplyTo);
}
}
catch (Exception exc)
{
_logger.Error(exc, "Unhandled exception in message pump");
}
}
private async Task Dispatch(BrokeredMessage message)
{
try
{
Exception exception = null;
try
{
_logger.Debug("Dispatching message: {0} from {1}", message, message.ReplyTo);
await _dispatcher.Dispatch(message);
_logger.Debug("Dispatched message: {0} from {1}", message, message.ReplyTo);
_logger.Debug("Completing message {0}", message);
await message.CompleteAsync();
_logger.Debug("Completed message {0}", message);
return;
}
catch (Exception exc)
{
exception = exc;
}
_logger.Error(exception, "Message dispatch failed");
try
{
_logger.Debug("Abandoning message {0} from {1}", message, message.ReplyTo);
await message.AbandonAsync(exception.ExceptionDetailsAsProperties(_clock.UtcNow));
_logger.Debug("Abandoned message {0} from {1}", message, message.ReplyTo);
}
catch (Exception exc)
{
_logger.Error(exc, "Could not call Abandon() on message {0} from {1}. Possible lock expiry?", message, message.ReplyTo);
}
}
catch (Exception exc)
{
_logger.Error(exc, "Unhandled exception in message pump");
}
}
async Task WaitAndAbandonMessage(BrokeredMessage message)
{
try
{
await _supervisor.Completed.ConfigureAwait(false);
await message.AbandonAsync().ConfigureAwait(false);
}
catch (Exception exception)
{
if (_log.IsErrorEnabled)
_log.Debug("Shutting down, abandoned message faulted: {_inputAddress}", exception);
}
}
async Task OnMessage(BrokeredMessage message)
{
if (_shuttingDown)
{
await WaitAndAbandonMessage(message);
return;
}
var current = Interlocked.Increment(ref _currentPendingDeliveryCount);
while (current > _maxPendingDeliveryCount)
Interlocked.CompareExchange(ref _maxPendingDeliveryCount, current, _maxPendingDeliveryCount);
var deliveryCount = Interlocked.Increment(ref _deliveryCount);
if (_log.IsDebugEnabled)
_log.DebugFormat("Receiving {0}:{1} - {2}", deliveryCount, message.MessageId, _receiveSettings.QueueDescription.Path);
var context = new ServiceBusReceiveContext(_inputAddress, message, _receiveObserver);
try
{
await _receiveObserver.PreReceive(context).ConfigureAwait(false);
await _receivePipe.Send(context).ConfigureAwait(false);
await context.CompleteTask.ConfigureAwait(false);
await message.CompleteAsync().ConfigureAwait(false);
await _receiveObserver.PostReceive(context).ConfigureAwait(false);
if (_log.IsDebugEnabled)
_log.DebugFormat("Receive completed: {0}", message.MessageId);
}
catch (Exception ex)
{
if (_log.IsErrorEnabled)
_log.Error($"Received faulted: {message.MessageId}", ex);
await message.AbandonAsync().ConfigureAwait(false);
await _receiveObserver.ReceiveFault(context, ex).ConfigureAwait(false);
}
finally
{
var pendingCount = Interlocked.Decrement(ref _currentPendingDeliveryCount);
if (pendingCount == 0 && _shuttingDown)
{
if (_log.IsDebugEnabled)
_log.DebugFormat("Receiver shutdown completed: {0}", _inputAddress);
_participant.SetComplete();
}
}
}
async Task OnMessage(BrokeredMessage message)
{
int current = Interlocked.Increment(ref _currentPendingDeliveryCount);
while (current > _maxPendingDeliveryCount)
Interlocked.CompareExchange(ref _maxPendingDeliveryCount, current, _maxPendingDeliveryCount);
long deliveryCount = Interlocked.Increment(ref _deliveryCount);
if (_log.IsDebugEnabled)
_log.DebugFormat("Receiving {0}:{1} - {2}", deliveryCount, message.MessageId, _receiveSettings.QueueDescription.Path);
var context = new ServiceBusReceiveContext(_inputAddress, message, _receiveObserver);
try
{
if (_shuttingDown)
{
await _completeTask.Task.ConfigureAwait(false);
throw new TransportException(_inputAddress, "Transport shutdown in progress, abandoning message");
}
await _receiveObserver.PreReceive(context).ConfigureAwait(false);
await _receivePipe.Send(context).ConfigureAwait(false);
await context.CompleteTask.ConfigureAwait(false);
await message.CompleteAsync().ConfigureAwait(false);
await _receiveObserver.PostReceive(context).ConfigureAwait(false);
if (_log.IsDebugEnabled)
_log.DebugFormat("Receive completed: {0}", message.MessageId);
}
catch (Exception ex)
{
if (_log.IsErrorEnabled)
_log.Error($"Received faulted: {message.MessageId}", ex);
await message.AbandonAsync().ConfigureAwait(false);
await _receiveObserver.ReceiveFault(context, ex).ConfigureAwait(false);
}
finally
{
int pendingCount = Interlocked.Decrement(ref _currentPendingDeliveryCount);
if (pendingCount == 0 && _shuttingDown)
_completeTask.TrySetResult(this);
}
}