Exemplo n.º 1
0
        /// <summary>
        /// Sends the provided <paramref name="command"></paramref> and waits for an event of <typeparamref name="TEvent"/> or exits if the specified timeout is expired.
        /// </summary>
        /// <param name="command">The <typeparamref name="TCommand"/> to send.</param>
        /// <param name="condition">A delegate to be executed over and over until it returns the <typeparamref name="TEvent"/> that is desired, return null to keep trying.</param>
        /// <param name="millisecondsTimeout">The number of milliseconds to wait, or <see cref="F:System.Threading.Timeout.Infinite"/> (-1) to wait indefinitely.</param>
        /// <param name="eventReceiver">If provided, is the <see cref="IEventReceiver{TAuthenticationToken}" /> that the event is expected to be returned on.</param>
        public virtual TEvent SendAndWait <TCommand, TEvent>(TCommand command, Func <IEnumerable <IEvent <TAuthenticationToken> >, TEvent> condition, int millisecondsTimeout, IEventReceiver <TAuthenticationToken> eventReceiver = null)
            where TCommand : ICommand <TAuthenticationToken>
        {
            DateTimeOffset startedAt      = DateTimeOffset.UtcNow;
            Stopwatch      mainStopWatch  = Stopwatch.StartNew();
            string         responseCode   = "200";
            bool           wasSuccessfull = false;

            IDictionary <string, string> telemetryProperties = new Dictionary <string, string> {
                { "Type", "InProcessBus" }
            };
            string telemetryName      = string.Format("{0}/{1}", command.GetType().FullName, command.Id);
            var    telemeteredCommand = command as ITelemeteredMessage;

            if (telemeteredCommand != null)
            {
                telemetryName = telemeteredCommand.TelemetryName;
            }
            telemetryName = string.Format("Command/{0}", telemetryName);

            TEvent result;

            try
            {
                if (eventReceiver != null)
                {
                    throw new NotSupportedException("Specifying a different event receiver is not yet supported.");
                }
                RouteHandlerDelegate commandHandler;
                if (!PrepareAndValidateCommand(command, out commandHandler))
                {
                    return((TEvent)(object)null);
                }

                result = (TEvent)(object)null;
                EventWaits.Add(command.CorrelationId, new List <IEvent <TAuthenticationToken> >());

                Action <IMessage> handler = commandHandler.Delegate;
                handler(command);
                Logger.LogInfo(string.Format("A command was sent of type {0}.", command.GetType().FullName));
                wasSuccessfull = true;
            }
            finally
            {
                mainStopWatch.Stop();
                TelemetryHelper.TrackDependency("InProcessBus/CommandBus", "Command", telemetryName, null, startedAt, mainStopWatch.Elapsed, responseCode, wasSuccessfull, telemetryProperties);
            }

            SpinWait.SpinUntil(() =>
            {
                IList <IEvent <TAuthenticationToken> > events = EventWaits[command.CorrelationId];

                result = condition(events);

                return(result != null);
            }, millisecondsTimeout, SpinWait.DefaultSleepInMilliseconds);

            TelemetryHelper.TrackDependency("InProcessBus/CommandBus", "Command/AndWait", string.Format("Command/AndWait{0}", telemetryName.Substring(7)), null, startedAt, mainStopWatch.Elapsed, responseCode, wasSuccessfull, telemetryProperties);
            return(result);
        }
Exemplo n.º 2
0
        /// <summary>
        /// Sends the provided <paramref name="command"></paramref> and waits for an event of <typeparamref name="TEvent"/> or exits if the specified timeout is expired.
        /// </summary>
        /// <param name="command">The <typeparamref name="TCommand"/> to send.</param>
        /// <param name="condition">A delegate to be executed over and over until it returns the <typeparamref name="TEvent"/> that is desired, return null to keep trying.</param>
        /// <param name="millisecondsTimeout">The number of milliseconds to wait, or <see cref="F:System.Threading.Timeout.Infinite"/> (-1) to wait indefinitely.</param>
        /// <param name="eventReceiver">If provided, is the <see cref="IEventReceiver{TAuthenticationToken}" /> that the event is expected to be returned on.</param>
        public virtual TEvent SendAndWait <TCommand, TEvent>(TCommand command, Func <IEnumerable <IEvent <TAuthenticationToken> >, TEvent> condition, int millisecondsTimeout, IEventReceiver <TAuthenticationToken> eventReceiver = null)
            where TCommand : ICommand <TAuthenticationToken>
        {
            if (eventReceiver != null)
            {
                throw new NotSupportedException("Specifying a different event receiver is not yet supported.");
            }
            RouteHandlerDelegate commandHandler;

            if (!PrepareAndValidateCommand(command, out commandHandler))
            {
                return((TEvent)(object)null);
            }

            TEvent result = (TEvent)(object)null;

            EventWaits.Add(command.CorrelationId, new List <IEvent <TAuthenticationToken> >());

            Action <IMessage> handler = commandHandler.Delegate;

            handler(command);

            SpinWait.SpinUntil(() =>
            {
                IList <IEvent <TAuthenticationToken> > events = EventWaits[command.CorrelationId];

                result = condition(events);

                return(result != null);
            }, millisecondsTimeout, SpinWait.DefaultSleepInMilliseconds);

            return(result);
        }
Exemplo n.º 3
0
        /// <summary>
        /// Sends the provided <paramref name="command"></paramref> and waits for an event of <typeparamref name="TEvent"/> or exits if the specified timeout is expired.
        /// </summary>
        /// <param name="command">The <typeparamref name="TCommand"/> to send.</param>
        /// <param name="condition">A delegate to be executed over and over until it returns the <typeparamref name="TEvent"/> that is desired, return null to keep trying.</param>
        /// <param name="millisecondsTimeout">The number of milliseconds to wait, or <see cref="F:System.Threading.Timeout.Infinite"/> (-1) to wait indefinitely.</param>
        /// <param name="eventReceiver">If provided, is the <see cref="IEventReceiver{TAuthenticationToken}" /> that the event is expected to be returned on.</param>
        public TEvent SendAndWait <TCommand, TEvent>(TCommand command, Func <IEnumerable <IEvent <TAuthenticationToken> >, TEvent> condition, int millisecondsTimeout,
                                                     IEventReceiver <TAuthenticationToken> eventReceiver = null) where TCommand : ICommand <TAuthenticationToken>
        {
            if (eventReceiver != null)
            {
                throw new NotSupportedException("Specifying a different event receiver is not yet supported.");
            }
            if (!PrepareAndValidateCommand(command))
            {
                return((TEvent)(object)null);
            }

            TEvent result = (TEvent)(object)null;

            EventWaits.Add(command.CorrelationId, new List <IEvent <TAuthenticationToken> >());

            ServiceBusPublisher.Send(new BrokeredMessage(MessageSerialiser.SerialiseCommand(command)));
            Logger.LogInfo(string.Format("A command was sent of type {0}.", command.GetType().FullName));

            SpinWait.SpinUntil(() =>
            {
                IList <IEvent <TAuthenticationToken> > events = EventWaits[command.CorrelationId];

                result = condition(events);

                return(result != null);
            }, millisecondsTimeout);

            return(result);
        }
Exemplo n.º 4
0
        protected virtual void ReceiveEvent(BrokeredMessage message)
        {
            var brokeredMessageRenewCancellationTokenSource = new CancellationTokenSource();

            try
            {
                Logger.LogDebug(string.Format("An event message arrived with the id '{0}'.", message.MessageId));
                string messageBody = message.GetBody <string>();

                IEvent <TAuthenticationToken> @event = AzureBusHelper.ReceiveEvent(messageBody, ReceiveEvent,
                                                                                   string.Format("id '{0}'", message.MessageId),
                                                                                   () =>
                {
                    // Remove message from queue
                    message.Complete();
                    Logger.LogDebug(string.Format("An event message arrived with the id '{0}' but processing was skipped due to event settings.", message.MessageId));
                },
                                                                                   () =>
                {
                    Task.Factory.StartNew(() =>
                    {
                        while (!brokeredMessageRenewCancellationTokenSource.Token.IsCancellationRequested)
                        {
                            //Based on LockedUntilUtc property to determine if the lock expires soon
                            if (DateTime.UtcNow > message.LockedUntilUtc.AddSeconds(-10))
                            {
                                // If so, repeat the message
                                message.RenewLock();
                            }

                            Thread.Sleep(500);
                        }
                    }, brokeredMessageRenewCancellationTokenSource.Token);
                }
                                                                                   );

                // Remove message from queue
                message.Complete();
                Logger.LogDebug(string.Format("An event message arrived and was processed with the id '{0}'.", message.MessageId));

                IList <IEvent <TAuthenticationToken> > events;
                if (EventWaits.TryGetValue(@event.CorrelationId, out events))
                {
                    events.Add(@event);
                }
            }
            catch (Exception exception)
            {
                // Indicates a problem, unlock message in queue
                Logger.LogError(string.Format("An event message arrived with the id '{0}' but failed to be process.", message.MessageId), exception: exception);
                message.Abandon();
            }
            finally
            {
                // Cancel the lock of renewing the task
                brokeredMessageRenewCancellationTokenSource.Cancel();
            }
        }
Exemplo n.º 5
0
        public virtual void Publish <TEvent>(TEvent @event)
            where TEvent : IEvent <TAuthenticationToken>
        {
            Type eventType = @event.GetType();

            if (@event.Frameworks != null && @event.Frameworks.Contains("Built-In"))
            {
                Logger.LogInfo("The provided event has already been processed by the Built-In bus.", string.Format("{0}\\PrepareAndValidateEvent({1})", GetType().FullName, eventType.FullName));
                return;
            }

            if (@event.AuthenticationToken == null || @event.AuthenticationToken.Equals(default(TAuthenticationToken)))
            {
                @event.AuthenticationToken = AuthenticationTokenHelper.GetAuthenticationToken();
            }
            @event.CorrelationId = CorrelationIdHelper.GetCorrelationId();

            if (string.IsNullOrWhiteSpace(@event.OriginatingFramework))
            {
                @event.TimeStamp            = DateTimeOffset.UtcNow;
                @event.OriginatingFramework = "Built-In";
            }

            var frameworks = new List <string>();

            if (@event.Frameworks != null)
            {
                frameworks.AddRange(@event.Frameworks);
            }
            frameworks.Add("Built-In");
            @event.Frameworks = frameworks;

            bool isRequired;

            if (!ConfigurationManager.TryGetSetting(string.Format("{0}.IsRequired", eventType.FullName), out isRequired))
            {
                isRequired = true;
            }

            IEnumerable <Action <IMessage> > handlers = Routes.GetHandlers(@event, isRequired).Select(x => x.Delegate).ToList();

            // This check doesn't require an isRequired check as there will be an exception raised above and handled below.
            if (!handlers.Any())
            {
                Logger.LogDebug(string.Format("An event handler for '{0}' is not required.", eventType.FullName));
            }

            foreach (Action <IMessage> handler in handlers)
            {
                IList <IEvent <TAuthenticationToken> > events;
                if (EventWaits.TryGetValue(@event.CorrelationId, out events))
                {
                    events.Add(@event);
                }
                handler(@event);
            }
        }
Exemplo n.º 6
0
        /// <summary>
        /// Sends the provided <paramref name="command"></paramref> and waits for an event of <typeparamref name="TEvent"/> or exits if the specified timeout is expired.
        /// </summary>
        /// <param name="command">The <typeparamref name="TCommand"/> to send.</param>
        /// <param name="condition">A delegate to be executed over and over until it returns the <typeparamref name="TEvent"/> that is desired, return null to keep trying.</param>
        /// <param name="millisecondsTimeout">The number of milliseconds to wait, or <see cref="F:System.Threading.Timeout.Infinite"/> (-1) to wait indefinitely.</param>
        /// <param name="eventReceiver">If provided, is the <see cref="IEventReceiver{TAuthenticationToken}" /> that the event is expected to be returned on.</param>
        public virtual TEvent SendAndWait <TCommand, TEvent>(TCommand command, Func <IEnumerable <IEvent <TAuthenticationToken> >, TEvent> condition, int millisecondsTimeout,
                                                             IEventReceiver <TAuthenticationToken> eventReceiver = null) where TCommand : ICommand <TAuthenticationToken>
        {
            if (eventReceiver != null)
            {
                throw new NotSupportedException("Specifying a different event receiver is not yet supported.");
            }
            if (!AzureBusHelper.PrepareAndValidateCommand(command, "Azure-ServiceBus"))
            {
                return((TEvent)(object)null);
            }

            TEvent result = (TEvent)(object)null;

            EventWaits.Add(command.CorrelationId, new List <IEvent <TAuthenticationToken> >());

            try
            {
                var brokeredMessage = new BrokeredMessage(MessageSerialiser.SerialiseCommand(command))
                {
                    CorrelationId = CorrelationIdHelper.GetCorrelationId().ToString("N")
                };
                brokeredMessage.Properties.Add("Type", command.GetType().FullName);
                PrivateServiceBusPublisher.Send(brokeredMessage);
            }
            catch (QuotaExceededException exception)
            {
                Logger.LogError("The size of the command being sent was too large.", exception: exception, metaData: new Dictionary <string, object> {
                    { "Command", command }
                });
                throw;
            }
            catch (Exception exception)
            {
                Logger.LogError("An issue occurred while trying to publish a command.", exception: exception, metaData: new Dictionary <string, object> {
                    { "Command", command }
                });
                throw;
            }
            Logger.LogInfo(string.Format("A command was sent of type {0}.", command.GetType().FullName));

            SpinWait.SpinUntil(() =>
            {
                IList <IEvent <TAuthenticationToken> > events = EventWaits[command.CorrelationId];

                result = condition(events);

                return(result != null);
            }, millisecondsTimeout, sleepInMilliseconds: 1000);

            return(result);
        }
Exemplo n.º 7
0
        /// <summary>
        /// Receives a <see cref="BrokeredMessage"/> from the event bus.
        /// </summary>
        protected virtual void ReceiveEvent(PartitionContext context, EventData eventData)
        {
            DateTimeOffset startedAt     = DateTimeOffset.UtcNow;
            Stopwatch      mainStopWatch = Stopwatch.StartNew();
            string         responseCode  = "200";
            // Null means it was skipped
            bool?              wasSuccessfull            = true;
            string             telemetryName             = string.Format("Cqrs/Handle/Event/{0}", eventData.SequenceNumber);
            ISingleSignOnToken authenticationToken       = null;
            Guid?              guidAuthenticationToken   = null;
            string             stringAuthenticationToken = null;
            int?intAuthenticationToken = null;

            IDictionary <string, string> telemetryProperties = new Dictionary <string, string> {
                { "Type", "Azure/EventHub" }
            };
            object value;

            if (eventData.Properties.TryGetValue("Type", out value))
            {
                telemetryProperties.Add("MessageType", value.ToString());
            }
            TelemetryHelper.TrackMetric("Cqrs/Handle/Event", CurrentHandles++, telemetryProperties);
            // Do a manual 10 try attempt with back-off
            for (int i = 0; i < 10; i++)
            {
                try
                {
                    Logger.LogDebug(string.Format("An event message arrived with the partition key '{0}', sequence number '{1}' and offset '{2}'.", eventData.PartitionKey, eventData.SequenceNumber, eventData.Offset));
                    string messageBody = Encoding.UTF8.GetString(eventData.GetBytes());

                    IEvent <TAuthenticationToken> @event = AzureBusHelper.ReceiveEvent(messageBody, ReceiveEvent,
                                                                                       string.Format("partition key '{0}', sequence number '{1}' and offset '{2}'", eventData.PartitionKey, eventData.SequenceNumber, eventData.Offset),
                                                                                       ExtractSignature(eventData),
                                                                                       SigningTokenConfigurationKey,
                                                                                       () =>
                    {
                        wasSuccessfull = null;
                        telemetryName  = string.Format("Cqrs/Handle/Event/Skipped/{0}", eventData.SequenceNumber);
                        responseCode   = "204";
                        // Remove message from queue
                        context.CheckpointAsync(eventData);
                        Logger.LogDebug(string.Format("An event message arrived with the partition key '{0}', sequence number '{1}' and offset '{2}' but processing was skipped due to event settings.", eventData.PartitionKey, eventData.SequenceNumber, eventData.Offset));
                        TelemetryHelper.TrackEvent("Cqrs/Handle/Event/Skipped", telemetryProperties);
                    }
                                                                                       );

                    if (wasSuccessfull != null)
                    {
                        if (@event != null)
                        {
                            telemetryName       = string.Format("{0}/{1}/{2}", @event.GetType().FullName, @event.GetIdentity(), @event.Id);
                            authenticationToken = @event.AuthenticationToken as ISingleSignOnToken;
                            if (AuthenticationTokenIsGuid)
                            {
                                guidAuthenticationToken = @event.AuthenticationToken as Guid?;
                            }
                            if (AuthenticationTokenIsString)
                            {
                                stringAuthenticationToken = @event.AuthenticationToken as string;
                            }
                            if (AuthenticationTokenIsInt)
                            {
                                intAuthenticationToken = @event.AuthenticationToken as int?;
                            }

                            var telemeteredMessage = @event as ITelemeteredMessage;
                            if (telemeteredMessage != null)
                            {
                                telemetryName = telemeteredMessage.TelemetryName;
                            }

                            telemetryName = string.Format("Cqrs/Handle/Event/{0}", telemetryName);
                        }
                        // Remove message from queue
                        context.CheckpointAsync(eventData);
                    }
                    Logger.LogDebug(string.Format("An event message arrived and was processed with the partition key '{0}', sequence number '{1}' and offset '{2}'.", eventData.PartitionKey, eventData.SequenceNumber, eventData.Offset));

                    IList <IEvent <TAuthenticationToken> > events;
                    if (EventWaits.TryGetValue(@event.CorrelationId, out events))
                    {
                        events.Add(@event);
                    }

                    wasSuccessfull = true;
                    responseCode   = "200";
                    return;
                }
                catch (UnAuthorisedMessageReceivedException exception)
                {
                    TelemetryHelper.TrackException(exception, null, telemetryProperties);
                    // Indicates a problem, unlock message in queue
                    Logger.LogError(string.Format("An event message arrived with the partition key '{0}', sequence number '{1}' and offset '{2}' but was not authorised.", eventData.PartitionKey, eventData.SequenceNumber, eventData.Offset), exception: exception);
                    wasSuccessfull = false;
                    responseCode   = "401";
                    telemetryProperties.Add("ExceptionType", exception.GetType().FullName);
                    telemetryProperties.Add("ExceptionMessage", exception.Message);
                }
                catch (NoHandlersRegisteredException exception)
                {
                    TelemetryHelper.TrackException(exception, null, telemetryProperties);
                    // Indicates a problem, unlock message in queue
                    Logger.LogError(string.Format("An event message arrived with the partition key '{0}', sequence number '{1}' and offset '{2}' but no handlers were found to process it.", eventData.PartitionKey, eventData.SequenceNumber, eventData.Offset), exception: exception);
                    wasSuccessfull = false;
                    responseCode   = "501";
                    telemetryProperties.Add("ExceptionType", exception.GetType().FullName);
                    telemetryProperties.Add("ExceptionMessage", exception.Message);
                }
                catch (NoHandlerRegisteredException exception)
                {
                    TelemetryHelper.TrackException(exception, null, telemetryProperties);
                    // Indicates a problem, unlock message in queue
                    Logger.LogError(string.Format("An event message arrived with the partition key '{0}', sequence number '{1}' and offset '{2}'s but no handler was found to process it.", eventData.PartitionKey, eventData.SequenceNumber, eventData.Offset), exception: exception);
                    wasSuccessfull = false;
                    responseCode   = "501";
                    telemetryProperties.Add("ExceptionType", exception.GetType().FullName);
                    telemetryProperties.Add("ExceptionMessage", exception.Message);
                }
                catch (Exception exception)
                {
                    // Indicates a problem, unlock message in queue
                    Logger.LogError(string.Format("An event message arrived with the partition key '{0}', sequence number '{1}' and offset '{2}' but failed to be process.", eventData.PartitionKey, eventData.SequenceNumber, eventData.Offset), exception: exception);

                    switch (i)
                    {
                    case 0:
                    case 1:
                        // 10 seconds
                        Thread.Sleep(10 * 1000);
                        break;

                    case 2:
                    case 3:
                        // 30 seconds
                        Thread.Sleep(30 * 1000);
                        break;

                    case 4:
                    case 5:
                    case 6:
                        // 1 minute
                        Thread.Sleep(60 * 1000);
                        break;

                    case 7:
                    case 8:
                        // 3 minutes
                        Thread.Sleep(3 * 60 * 1000);
                        break;

                    case 9:
                        telemetryProperties.Add("ExceptionType", exception.GetType().FullName);
                        telemetryProperties.Add("ExceptionMessage", exception.Message);
                        break;
                    }
                    wasSuccessfull = false;
                    responseCode   = "500";
                }
                finally
                {
                    // Eventually just accept it
                    context.CheckpointAsync(eventData);

                    TelemetryHelper.TrackMetric("Cqrs/Handle/Event", CurrentHandles--, telemetryProperties);

                    mainStopWatch.Stop();
                    if (guidAuthenticationToken != null)
                    {
                        TelemetryHelper.TrackRequest
                        (
                            telemetryName,
                            guidAuthenticationToken,
                            startedAt,
                            mainStopWatch.Elapsed,
                            responseCode,
                            wasSuccessfull == null || wasSuccessfull.Value,
                            telemetryProperties
                        );
                    }
                    else if (intAuthenticationToken != null)
                    {
                        TelemetryHelper.TrackRequest
                        (
                            telemetryName,
                            intAuthenticationToken,
                            startedAt,
                            mainStopWatch.Elapsed,
                            responseCode,
                            wasSuccessfull == null || wasSuccessfull.Value,
                            telemetryProperties
                        );
                    }
                    else if (stringAuthenticationToken != null)
                    {
                        TelemetryHelper.TrackRequest
                        (
                            telemetryName,
                            stringAuthenticationToken,
                            startedAt,
                            mainStopWatch.Elapsed,
                            responseCode,
                            wasSuccessfull == null || wasSuccessfull.Value,
                            telemetryProperties
                        );
                    }
                    else
                    {
                        TelemetryHelper.TrackRequest
                        (
                            telemetryName,
                            authenticationToken,
                            startedAt,
                            mainStopWatch.Elapsed,
                            responseCode,
                            wasSuccessfull == null || wasSuccessfull.Value,
                            telemetryProperties
                        );
                    }

                    TelemetryHelper.Flush();
                }
            }
        }
Exemplo n.º 8
0
        protected virtual void ReceiveEvent(BrokeredMessage message)
        {
            var brokeredMessageRenewCancellationTokenSource = new CancellationTokenSource();

            try
            {
                Logger.LogDebug(string.Format("An event message arrived with the id '{0}'.", message.MessageId));
                string messageBody = message.GetBody <string>();
                IEvent <TAuthenticationToken> @event;

                try
                {
                    @event = MessageSerialiser.DeserialiseEvent(messageBody);
                }
                catch (JsonSerializationException exception)
                {
                    JsonSerializationException checkException = exception;
                    bool safeToExit = false;
                    do
                    {
                        if (checkException.Message.StartsWith("Could not load assembly"))
                        {
                            safeToExit = true;
                            break;
                        }
                    } while ((checkException = checkException.InnerException as JsonSerializationException) != null);
                    if (safeToExit)
                    {
                        const string pattern = @"(?<=^Error resolving type specified in JSON ').+?(?='\. Path '\$type')";
                        Match        match   = new Regex(pattern).Match(exception.Message);
                        if (match.Success)
                        {
                            string[] typeParts = match.Value.Split(',');
                            if (typeParts.Length == 2)
                            {
                                string classType  = typeParts[0];
                                bool   isRequired = BusHelper.IsEventRequired(string.Format("{0}.IsRequired", classType));

                                if (!isRequired)
                                {
                                    // Remove message from queue
                                    message.Complete();
                                    Logger.LogDebug(string.Format("An event message arrived with the id '{0}' but processing was skipped due to event settings.", message.MessageId));
                                    return;
                                }
                            }
                        }
                    }
                    throw;
                }

                CorrelationIdHelper.SetCorrelationId(@event.CorrelationId);
                Logger.LogInfo(string.Format("An event message arrived with the id '{0}' was of type {1}.", message.MessageId, @event.GetType().FullName));

                bool canRefresh;
                if (!ConfigurationManager.TryGetSetting(string.Format("{0}.ShouldRefresh", @event.GetType().FullName), out canRefresh))
                {
                    canRefresh = false;
                }

                if (canRefresh)
                {
                    Task.Factory.StartNew(() =>
                    {
                        while (!brokeredMessageRenewCancellationTokenSource.Token.IsCancellationRequested)
                        {
                            //Based on LockedUntilUtc property to determine if the lock expires soon
                            if (DateTime.UtcNow > message.LockedUntilUtc.AddSeconds(-10))
                            {
                                // If so, repeat the message
                                message.RenewLock();
                            }

                            Thread.Sleep(500);
                        }
                    }, brokeredMessageRenewCancellationTokenSource.Token);
                }

                ReceiveEvent(@event);

                // Remove message from queue
                message.Complete();
                Logger.LogDebug(string.Format("An event message arrived and was processed with the id '{0}'.", message.MessageId));

                IList <IEvent <TAuthenticationToken> > events;
                if (EventWaits.TryGetValue(@event.CorrelationId, out events))
                {
                    events.Add(@event);
                }
            }
            catch (Exception exception)
            {
                // Indicates a problem, unlock message in queue
                Logger.LogError(string.Format("An event message arrived with the id '{0}' but failed to be process.", message.MessageId), exception: exception);
                message.Abandon();
            }
            finally
            {
                // Cancel the lock of renewing the task
                brokeredMessageRenewCancellationTokenSource.Cancel();
            }
        }
Exemplo n.º 9
0
        /// <summary>
        /// Publishes the provided <paramref name="command"></paramref> and waits for an event of <typeparamref name="TEvent"/> or exits if the specified timeout is expired.
        /// </summary>
        /// <param name="command">The <typeparamref name="TCommand"/> to publish.</param>
        /// <param name="condition">A delegate to be executed over and over until it returns the <typeparamref name="TEvent"/> that is desired, return null to keep trying.</param>
        /// <param name="millisecondsTimeout">The number of milliseconds to wait, or <see cref="F:System.Threading.Timeout.Infinite"/> (-1) to wait indefinitely.</param>
        /// <param name="eventReceiver">If provided, is the <see cref="IEventReceiver{TAuthenticationToken}" /> that the event is expected to be returned on.</param>
        public virtual TEvent PublishAndWait <TCommand, TEvent>(TCommand command, Func <IEnumerable <IEvent <TAuthenticationToken> >, TEvent> condition, int millisecondsTimeout,
                                                                IEventReceiver <TAuthenticationToken> eventReceiver = null) where TCommand : ICommand <TAuthenticationToken>
        {
            DateTimeOffset startedAt      = DateTimeOffset.UtcNow;
            Stopwatch      mainStopWatch  = Stopwatch.StartNew();
            string         responseCode   = "200";
            bool           wasSuccessfull = false;

            IDictionary <string, string> telemetryProperties = new Dictionary <string, string> {
                { "Type", "Azure/Servicebus" }
            };
            string telemetryName      = string.Format("{0}/{1}", command.GetType().FullName, command.Id);
            var    telemeteredCommand = command as ITelemeteredMessage;

            if (telemeteredCommand != null)
            {
                telemetryName = telemeteredCommand.TelemetryName;
            }
            telemetryName = string.Format("Command/{0}", telemetryName);

            TEvent result;

            try
            {
                if (eventReceiver != null)
                {
                    throw new NotSupportedException("Specifying a different event receiver is not yet supported.");
                }
                if (!AzureBusHelper.PrepareAndValidateCommand(command, "Azure-ServiceBus"))
                {
                    return((TEvent)(object)null);
                }

                result = (TEvent)(object)null;
                EventWaits.Add(command.CorrelationId, new List <IEvent <TAuthenticationToken> >());

                try
                {
                    var brokeredMessage = new BrokeredMessage(MessageSerialiser.SerialiseCommand(command))
                    {
                        CorrelationId = CorrelationIdHelper.GetCorrelationId().ToString("N")
                    };
                    brokeredMessage.Properties.Add("Type", command.GetType().FullName);
                    PrivateServiceBusPublisher.Send(brokeredMessage);
                }
                catch (QuotaExceededException exception)
                {
                    responseCode = "429";
                    Logger.LogError("The size of the command being sent was too large.", exception: exception, metaData: new Dictionary <string, object> {
                        { "Command", command }
                    });
                    throw;
                }
                catch (Exception exception)
                {
                    responseCode = "500";
                    Logger.LogError("An issue occurred while trying to publish a command.", exception: exception, metaData: new Dictionary <string, object> {
                        { "Command", command }
                    });
                    throw;
                }
                Logger.LogInfo(string.Format("A command was sent of type {0}.", command.GetType().FullName));
                wasSuccessfull = true;
            }
            finally
            {
                mainStopWatch.Stop();
                TelemetryHelper.TrackDependency("Azure/Servicebus/CommandBus", "Command", telemetryName, null, startedAt, mainStopWatch.Elapsed, responseCode, wasSuccessfull, telemetryProperties);
            }

            SpinWait.SpinUntil(() =>
            {
                IList <IEvent <TAuthenticationToken> > events = EventWaits[command.CorrelationId];

                result = condition(events);

                return(result != null);
            }, millisecondsTimeout, sleepInMilliseconds: 1000);

            TelemetryHelper.TrackDependency("Azure/Servicebus/CommandBus", "Command/AndWait", string.Format("Command/AndWait{0}", telemetryName.Substring(7)), null, startedAt, mainStopWatch.Elapsed, responseCode, wasSuccessfull, telemetryProperties);
            return(result);
        }
Exemplo n.º 10
0
        protected virtual void ReceiveEvent(IMessageReceiver client, BrokeredMessage message)
#endif
        {
            DateTimeOffset startedAt     = DateTimeOffset.UtcNow;
            Stopwatch      mainStopWatch = Stopwatch.StartNew();
            string         responseCode  = "200";
            // Null means it was skipped
            bool?              wasSuccessfull            = true;
            string             telemetryName             = string.Format("Cqrs/Handle/Event/{0}", message.MessageId);
            ISingleSignOnToken authenticationToken       = null;
            Guid?              guidAuthenticationToken   = null;
            string             stringAuthenticationToken = null;
            int?intAuthenticationToken = null;

            IDictionary <string, string> telemetryProperties = ExtractTelemetryProperties(message, "Azure/Servicebus");

            TelemetryHelper.TrackMetric("Cqrs/Handle/Event", CurrentHandles++, telemetryProperties);
            var brokeredMessageRenewCancellationTokenSource = new CancellationTokenSource();

            try
            {
                try
                {
                    Logger.LogDebug(string.Format("An event message arrived with the id '{0}'.", message.MessageId));
                    string messageBody = message.GetBodyAsString();

                    IEvent <TAuthenticationToken> @event = AzureBusHelper.ReceiveEvent(messageBody, ReceiveEvent,
                                                                                       string.Format("id '{0}'", message.MessageId),
                                                                                       ExtractSignature(message),
                                                                                       SigningTokenConfigurationKey,
                                                                                       () =>
                    {
                        wasSuccessfull = null;
                        telemetryName  = string.Format("Cqrs/Handle/Event/Skipped/{0}", message.MessageId);
                        responseCode   = "204";
                        // Remove message from queue
                        try
                        {
#if NET452
                            message.Complete();
#endif
#if NETSTANDARD2_0
                            client.CompleteAsync(message.SystemProperties.LockToken).Wait(1500);
#endif
                        }
                        catch (AggregateException aggregateException)
                        {
                            if (aggregateException.InnerException is MessageLockLostException)
                            {
                                throw new MessageLockLostException(string.Format("The lock supplied for the skipped event message '{0}' is invalid.", message.MessageId), aggregateException.InnerException);
                            }
                            else
                            {
                                throw;
                            }
                        }
                        catch (MessageLockLostException exception)
                        {
                            throw new MessageLockLostException(string.Format("The lock supplied for the skipped event message '{0}' is invalid.", message.MessageId), exception);
                        }
                        Logger.LogDebug(string.Format("An event message arrived with the id '{0}' but processing was skipped due to event settings.", message.MessageId));
                        TelemetryHelper.TrackEvent("Cqrs/Handle/Event/Skipped", telemetryProperties);
                    },
                                                                                       () =>
                    {
#if NET452
                        AzureBusHelper.RefreshLock(brokeredMessageRenewCancellationTokenSource, message, "event");
#endif
#if NETSTANDARD2_0
                        AzureBusHelper.RefreshLock(client, brokeredMessageRenewCancellationTokenSource, message, "event");
#endif
                    }
                                                                                       );

                    if (wasSuccessfull != null)
                    {
                        if (@event != null)
                        {
                            telemetryName       = string.Format("{0}/{1}/{2}", @event.GetType().FullName, @event.GetIdentity(), @event.Id);
                            authenticationToken = @event.AuthenticationToken as ISingleSignOnToken;
                            if (AuthenticationTokenIsGuid)
                            {
                                guidAuthenticationToken = @event.AuthenticationToken as Guid?;
                            }
                            if (AuthenticationTokenIsString)
                            {
                                stringAuthenticationToken = @event.AuthenticationToken as string;
                            }
                            if (AuthenticationTokenIsInt)
                            {
                                intAuthenticationToken = @event.AuthenticationToken as int?;
                            }

                            var telemeteredMessage = @event as ITelemeteredMessage;
                            if (telemeteredMessage != null)
                            {
                                telemetryName = telemeteredMessage.TelemetryName;
                            }

                            telemetryName = string.Format("Cqrs/Handle/Event/{0}", telemetryName);
                        }
                        // Remove message from queue
                        try
                        {
#if NET452
                            message.Complete();
#endif
#if NETSTANDARD2_0
                            client.CompleteAsync(message.SystemProperties.LockToken).Wait(1500);
#endif
                        }
                        catch (AggregateException aggregateException)
                        {
                            if (aggregateException.InnerException is MessageLockLostException)
                            {
                                throw new MessageLockLostException(string.Format("The lock supplied for event '{0}' of type {1} is invalid.", @event.Id, @event.GetType().Name), aggregateException.InnerException);
                            }
                            else
                            {
                                throw;
                            }
                        }
                        catch (MessageLockLostException exception)
                        {
                            throw new MessageLockLostException(string.Format("The lock supplied for event '{0}' of type {1} is invalid.", @event.Id, @event.GetType().Name), exception);
                        }
                    }
                    Logger.LogDebug(string.Format("An event message arrived and was processed with the id '{0}'.", message.MessageId));

                    IList <IEvent <TAuthenticationToken> > events;
                    if (EventWaits.TryGetValue(@event.CorrelationId, out events))
                    {
                        events.Add(@event);
                    }
                }
                catch (AggregateException aggregateException)
                {
                    throw aggregateException.InnerException;
                }
            }
            catch (MessageLockLostException exception)
            {
                IDictionary <string, string> subTelemetryProperties = new Dictionary <string, string>(telemetryProperties);
                subTelemetryProperties.Add("TimeTaken", mainStopWatch.Elapsed.ToString());
                TelemetryHelper.TrackException(exception, null, subTelemetryProperties);
                if (ThrowExceptionOnReceiverMessageLockLostExceptionDuringComplete)
                {
                    Logger.LogError(exception.Message, exception: exception);
                    // Indicates a problem, unlock message in queue
#if NET452
                    message.Abandon();
#endif
#if NETSTANDARD2_0
                    client.AbandonAsync(message.SystemProperties.LockToken).Wait(1500);
#endif
                    wasSuccessfull = false;
                }
                else
                {
                    Logger.LogWarning(exception.Message, exception: exception);
                    try
                    {
#if NET452
                        message.DeadLetter("LockLostButHandled", "The message was handled but the lock was lost.");
#endif
#if NETSTANDARD2_0
                        client.DeadLetterAsync(message.SystemProperties.LockToken, "LockLostButHandled", "The message was handled but the lock was lost.").Wait(1500);
#endif
                    }
                    catch (Exception)
                    {
                        // Oh well, move on.
#if NET452
                        message.Abandon();
#endif
#if NETSTANDARD2_0
                        client.AbandonAsync(message.SystemProperties.LockToken).Wait(1500);
#endif
                    }
                }
                responseCode = "599";
            }
            catch (UnAuthorisedMessageReceivedException exception)
            {
                TelemetryHelper.TrackException(exception, null, telemetryProperties);
                // Indicates a problem, unlock message in queue
                Logger.LogError(string.Format("An event message arrived with the id '{0}' but was not authorised.", message.MessageId), exception: exception);
#if NET452
                message.DeadLetter("UnAuthorisedMessageReceivedException", exception.Message);
#endif
#if NETSTANDARD2_0
                client.DeadLetterAsync(message.SystemProperties.LockToken, "UnAuthorisedMessageReceivedException", exception.Message).Wait(1500);
#endif
                wasSuccessfull = false;
                responseCode   = "401";
                telemetryProperties.Add("ExceptionType", exception.GetType().FullName);
                telemetryProperties.Add("ExceptionMessage", exception.Message);
            }
            catch (NoHandlersRegisteredException exception)
            {
                TelemetryHelper.TrackException(exception, null, telemetryProperties);
                // Indicates a problem, unlock message in queue
                Logger.LogError(string.Format("An event message arrived with the id '{0}' but no handlers were found to process it.", message.MessageId), exception: exception);
#if NET452
                message.DeadLetter("NoHandlersRegisteredException", exception.Message);
#endif
#if NETSTANDARD2_0
                client.DeadLetterAsync(message.SystemProperties.LockToken, "NoHandlersRegisteredException", exception.Message).Wait(1500);
#endif
                wasSuccessfull = false;
                responseCode   = "501";
                telemetryProperties.Add("ExceptionType", exception.GetType().FullName);
                telemetryProperties.Add("ExceptionMessage", exception.Message);
            }
            catch (NoHandlerRegisteredException exception)
            {
                TelemetryHelper.TrackException(exception, null, telemetryProperties);
                // Indicates a problem, unlock message in queue
                Logger.LogError(string.Format("An event message arrived with the id '{0}' but no handler was found to process it.", message.MessageId), exception: exception);
#if NET452
                message.DeadLetter("NoHandlerRegisteredException", exception.Message);
#endif
#if NETSTANDARD2_0
                client.DeadLetterAsync(message.SystemProperties.LockToken, "NoHandlerRegisteredException", exception.Message).Wait(1500);
#endif
                wasSuccessfull = false;
                responseCode   = "501";
                telemetryProperties.Add("ExceptionType", exception.GetType().FullName);
                telemetryProperties.Add("ExceptionMessage", exception.Message);
            }
            catch (Exception exception)
            {
                TelemetryHelper.TrackException(exception, null, telemetryProperties);
                // Indicates a problem, unlock message in queue
                Logger.LogError(string.Format("An event message arrived with the id '{0}' but failed to be process.", message.MessageId), exception: exception);
#if NET452
                message.Abandon();
#endif
#if NETSTANDARD2_0
                client.AbandonAsync(message.SystemProperties.LockToken).Wait(1500);
#endif
                wasSuccessfull = false;
                responseCode   = "500";
                telemetryProperties.Add("ExceptionType", exception.GetType().FullName);
                telemetryProperties.Add("ExceptionMessage", exception.Message);
            }
            finally
            {
                // Cancel the lock of renewing the task
                brokeredMessageRenewCancellationTokenSource.Cancel();
                TelemetryHelper.TrackMetric("Cqrs/Handle/Event", CurrentHandles--, telemetryProperties);

                mainStopWatch.Stop();
                if (guidAuthenticationToken != null)
                {
                    TelemetryHelper.TrackRequest
                    (
                        telemetryName,
                        guidAuthenticationToken,
                        startedAt,
                        mainStopWatch.Elapsed,
                        responseCode,
                        wasSuccessfull == null || wasSuccessfull.Value,
                        telemetryProperties
                    );
                }
                else if (intAuthenticationToken != null)
                {
                    TelemetryHelper.TrackRequest
                    (
                        telemetryName,
                        intAuthenticationToken,
                        startedAt,
                        mainStopWatch.Elapsed,
                        responseCode,
                        wasSuccessfull == null || wasSuccessfull.Value,
                        telemetryProperties
                    );
                }
                else if (stringAuthenticationToken != null)
                {
                    TelemetryHelper.TrackRequest
                    (
                        telemetryName,
                        stringAuthenticationToken,
                        startedAt,
                        mainStopWatch.Elapsed,
                        responseCode,
                        wasSuccessfull == null || wasSuccessfull.Value,
                        telemetryProperties
                    );
                }
                else
                {
                    TelemetryHelper.TrackRequest
                    (
                        telemetryName,
                        authenticationToken,
                        startedAt,
                        mainStopWatch.Elapsed,
                        responseCode,
                        wasSuccessfull == null || wasSuccessfull.Value,
                        telemetryProperties
                    );
                }

                TelemetryHelper.Flush();
            }
        }
Exemplo n.º 11
0
        public virtual void Publish <TEvent>(TEvent @event)
            where TEvent : IEvent <TAuthenticationToken>
        {
            DateTimeOffset startedAt      = DateTimeOffset.UtcNow;
            Stopwatch      mainStopWatch  = Stopwatch.StartNew();
            string         responseCode   = "200";
            bool           wasSuccessfull = false;

            IDictionary <string, string> telemetryProperties = new Dictionary <string, string> {
                { "Type", "InProcessBus" }
            };
            string telemetryName    = string.Format("{0}/{1}", @event.GetType().FullName, @event.Id);
            var    telemeteredEvent = @event as ITelemeteredMessage;

            if (telemeteredEvent != null)
            {
                telemetryName = telemeteredEvent.TelemetryName;
            }
            telemetryName = string.Format("Event/{0}", telemetryName);

            try
            {
                Type eventType = @event.GetType();

                if (@event.Frameworks != null && @event.Frameworks.Contains("Built-In"))
                {
                    Logger.LogInfo("The provided event has already been processed by the Built-In bus.", string.Format("{0}\\PrepareAndValidateEvent({1})", GetType().FullName, eventType.FullName));
                    return;
                }

                if (@event.AuthenticationToken == null || @event.AuthenticationToken.Equals(default(TAuthenticationToken)))
                {
                    @event.AuthenticationToken = AuthenticationTokenHelper.GetAuthenticationToken();
                }
                @event.CorrelationId = CorrelationIdHelper.GetCorrelationId();

                if (string.IsNullOrWhiteSpace(@event.OriginatingFramework))
                {
                    @event.TimeStamp            = DateTimeOffset.UtcNow;
                    @event.OriginatingFramework = "Built-In";
                }

                var frameworks = new List <string>();
                if (@event.Frameworks != null)
                {
                    frameworks.AddRange(@event.Frameworks);
                }
                frameworks.Add("Built-In");
                @event.Frameworks = frameworks;

                bool isRequired;
                if (!ConfigurationManager.TryGetSetting(string.Format("{0}.IsRequired", eventType.FullName), out isRequired))
                {
                    isRequired = true;
                }

                IEnumerable <Action <IMessage> > handlers = Routes.GetHandlers(@event, isRequired).Select(x => x.Delegate).ToList();
                // This check doesn't require an isRequired check as there will be an exception raised above and handled below.
                if (!handlers.Any())
                {
                    Logger.LogDebug(string.Format("An event handler for '{0}' is not required.", eventType.FullName));
                }

                foreach (Action <IMessage> handler in handlers)
                {
                    IList <IEvent <TAuthenticationToken> > events;
                    if (EventWaits.TryGetValue(@event.CorrelationId, out events))
                    {
                        events.Add(@event);
                    }
                    handler(@event);
                }

                Logger.LogInfo(string.Format("An event was sent of type {0}.", eventType.FullName));
                wasSuccessfull = true;
            }
            catch (Exception exception)
            {
                responseCode = "500";
                Logger.LogError("An issue occurred while trying to publish an event.", exception: exception, metaData: new Dictionary <string, object> {
                    { "Event", @event }
                });
                throw;
            }
            finally
            {
                mainStopWatch.Stop();
                TelemetryHelper.TrackDependency("InProcessBus/EventBus", "Event", telemetryName, null, startedAt, mainStopWatch.Elapsed, responseCode, wasSuccessfull, telemetryProperties);
            }
        }
Exemplo n.º 12
0
        protected virtual void ReceiveEvent(BrokeredMessage message)
        {
            DateTimeOffset startedAt     = DateTimeOffset.UtcNow;
            Stopwatch      mainStopWatch = Stopwatch.StartNew();
            string         responseCode  = "200";
            // Null means it was skipped
            bool?              wasSuccessfull      = true;
            string             telemetryName       = string.Format("Cqrs/Handle/Event/{0}", message.MessageId);
            ISingleSignOnToken authenticationToken = null;

            IDictionary <string, string> telemetryProperties = new Dictionary <string, string> {
                { "Type", "Azure/Servicebus" }
            };
            object value;

            if (message.Properties.TryGetValue("Type", out value))
            {
                telemetryProperties.Add("MessageType", value.ToString());
            }
            TelemetryHelper.TrackMetric("Cqrs/Handle/Event", CurrentHandles++, telemetryProperties);
            var brokeredMessageRenewCancellationTokenSource = new CancellationTokenSource();

            try
            {
                Logger.LogDebug(string.Format("An event message arrived with the id '{0}'.", message.MessageId));
                string messageBody = message.GetBody <string>();

                IEvent <TAuthenticationToken> @event = AzureBusHelper.ReceiveEvent(messageBody, ReceiveEvent,
                                                                                   string.Format("id '{0}'", message.MessageId),
                                                                                   () =>
                {
                    wasSuccessfull = null;
                    telemetryName  = string.Format("Cqrs/Handle/Event/Skipped/{0}", message.MessageId);
                    responseCode   = "204";
                    // Remove message from queue
                    try
                    {
                        message.Complete();
                    }
                    catch (MessageLockLostException exception)
                    {
                        throw new MessageLockLostException(string.Format("The lock supplied for the skipped event message '{0}' is invalid.", message.MessageId), exception);
                    }
                    Logger.LogDebug(string.Format("An event message arrived with the id '{0}' but processing was skipped due to event settings.", message.MessageId));
                    TelemetryHelper.TrackEvent("Cqrs/Handle/Event/Skipped", telemetryProperties);
                },
                                                                                   () =>
                {
                    AzureBusHelper.RefreshLock(brokeredMessageRenewCancellationTokenSource, message, "event");
                }
                                                                                   );

                if (wasSuccessfull != null)
                {
                    if (@event != null)
                    {
                        telemetryName       = string.Format("{0}/{1}", @event.GetType().FullName, @event.Id);
                        authenticationToken = @event.AuthenticationToken as ISingleSignOnToken;

                        var telemeteredMessage = @event as ITelemeteredMessage;
                        if (telemeteredMessage != null)
                        {
                            telemetryName = telemeteredMessage.TelemetryName;
                        }

                        telemetryName = string.Format("Cqrs/Handle/Event/{0}", telemetryName);
                    }
                    // Remove message from queue
                    try
                    {
                        message.Complete();
                    }
                    catch (MessageLockLostException exception)
                    {
                        throw new MessageLockLostException(string.Format("The lock supplied for event '{0}' of type {1} is invalid.", @event.Id, @event.GetType().Name), exception);
                    }
                }
                Logger.LogDebug(string.Format("An event message arrived and was processed with the id '{0}'.", message.MessageId));

                IList <IEvent <TAuthenticationToken> > events;
                if (EventWaits.TryGetValue(@event.CorrelationId, out events))
                {
                    events.Add(@event);
                }
            }
            catch (Exception exception)
            {
                TelemetryHelper.TrackException(exception, null, telemetryProperties);
                // Indicates a problem, unlock message in queue
                Logger.LogError(string.Format("An event message arrived with the id '{0}' but failed to be process.", message.MessageId), exception: exception);
                message.Abandon();
                wasSuccessfull = false;
                responseCode   = "500";
            }
            finally
            {
                // Cancel the lock of renewing the task
                brokeredMessageRenewCancellationTokenSource.Cancel();
                TelemetryHelper.TrackMetric("Cqrs/Handle/Event", CurrentHandles--, telemetryProperties);

                mainStopWatch.Stop();
                TelemetryHelper.TrackRequest
                (
                    telemetryName,
                    authenticationToken,
                    startedAt,
                    mainStopWatch.Elapsed,
                    responseCode,
                    wasSuccessfull == null || wasSuccessfull.Value,
                    telemetryProperties
                );

                TelemetryHelper.Flush();
            }
        }
Exemplo n.º 13
0
        protected virtual void ReceiveEvent(PartitionContext context, EventData eventData)
        {
            IDictionary <string, string> telemetryProperties = new Dictionary <string, string> {
                { "Type", "Azure/EventHub" }
            };

            TelemetryHelper.TrackMetric("Cqrs/Handle/Event", CurrentHandles++, telemetryProperties);
            // Do a manual 10 try attempt with back-off
            for (int i = 0; i < 10; i++)
            {
                try
                {
                    Logger.LogDebug(string.Format("An event message arrived with the partition key '{0}', sequence number '{1}' and offset '{2}'.", eventData.PartitionKey, eventData.SequenceNumber, eventData.Offset));
                    string messageBody = Encoding.UTF8.GetString(eventData.GetBytes());

                    IEvent <TAuthenticationToken> @event = AzureBusHelper.ReceiveEvent(messageBody, ReceiveEvent,
                                                                                       string.Format("partition key '{0}', sequence number '{1}' and offset '{2}'", eventData.PartitionKey, eventData.SequenceNumber, eventData.Offset),
                                                                                       () =>
                    {
                        // Remove message from queue
                        context.CheckpointAsync(eventData);
                        Logger.LogDebug(string.Format("An event message arrived with the partition key '{0}', sequence number '{1}' and offset '{2}' but processing was skipped due to event settings.", eventData.PartitionKey, eventData.SequenceNumber, eventData.Offset));
                    }
                                                                                       );

                    // Remove message from queue
                    context.CheckpointAsync(eventData);
                    Logger.LogDebug(string.Format("An event message arrived and was processed with the partition key '{0}', sequence number '{1}' and offset '{2}'.", eventData.PartitionKey, eventData.SequenceNumber, eventData.Offset));

                    IList <IEvent <TAuthenticationToken> > events;
                    if (EventWaits.TryGetValue(@event.CorrelationId, out events))
                    {
                        events.Add(@event);
                    }

                    return;
                }
                catch (Exception exception)
                {
                    // Indicates a problem, unlock message in queue
                    Logger.LogError(string.Format("An event message arrived with the partition key '{0}', sequence number '{1}' and offset '{2}' but failed to be process.", eventData.PartitionKey, eventData.SequenceNumber, eventData.Offset), exception: exception);

                    switch (i)
                    {
                    case 0:
                    case 1:
                        // 10 seconds
                        Thread.Sleep(10 * 1000);
                        break;

                    case 2:
                    case 3:
                        // 30 seconds
                        Thread.Sleep(30 * 1000);
                        break;

                    case 4:
                    case 5:
                    case 6:
                        // 1 minute
                        Thread.Sleep(60 * 1000);
                        break;

                    case 7:
                    case 8:
                    case 9:
                        // 3 minutes
                        Thread.Sleep(3 * 60 * 1000);
                        break;
                    }
                }
            }
            // Eventually just accept it
            context.CheckpointAsync(eventData);
            TelemetryHelper.TrackMetric("Cqrs/Handle/Event", CurrentHandles--, telemetryProperties);
        }