protected virtual void ReceiveEvent(BrokeredMessage message) { var brokeredMessageRenewCancellationTokenSource = new CancellationTokenSource(); try { Logger.LogDebug(string.Format("An event message arrived with the id '{0}'.", message.MessageId)); string messageBody = message.GetBody <string>(); IEvent <TAuthenticationToken> @event = AzureBusHelper.ReceiveEvent(messageBody, ReceiveEvent, string.Format("id '{0}'", message.MessageId), () => { // Remove message from queue message.Complete(); Logger.LogDebug(string.Format("An event message arrived with the id '{0}' but processing was skipped due to event settings.", message.MessageId)); }, () => { Task.Factory.StartNew(() => { while (!brokeredMessageRenewCancellationTokenSource.Token.IsCancellationRequested) { //Based on LockedUntilUtc property to determine if the lock expires soon if (DateTime.UtcNow > message.LockedUntilUtc.AddSeconds(-10)) { // If so, repeat the message message.RenewLock(); } Thread.Sleep(500); } }, brokeredMessageRenewCancellationTokenSource.Token); } ); // Remove message from queue message.Complete(); Logger.LogDebug(string.Format("An event message arrived and was processed with the id '{0}'.", message.MessageId)); IList <IEvent <TAuthenticationToken> > events; if (EventWaits.TryGetValue(@event.CorrelationId, out events)) { events.Add(@event); } } catch (Exception exception) { // Indicates a problem, unlock message in queue Logger.LogError(string.Format("An event message arrived with the id '{0}' but failed to be process.", message.MessageId), exception: exception); message.Abandon(); } finally { // Cancel the lock of renewing the task brokeredMessageRenewCancellationTokenSource.Cancel(); } }
/// <summary> /// Receives a <see cref="BrokeredMessage"/> from the event bus. /// </summary> protected virtual void ReceiveEvent(PartitionContext context, EventData eventData) { DateTimeOffset startedAt = DateTimeOffset.UtcNow; Stopwatch mainStopWatch = Stopwatch.StartNew(); string responseCode = "200"; // Null means it was skipped bool? wasSuccessfull = true; string telemetryName = string.Format("Cqrs/Handle/Event/{0}", eventData.SequenceNumber); ISingleSignOnToken authenticationToken = null; Guid? guidAuthenticationToken = null; string stringAuthenticationToken = null; int?intAuthenticationToken = null; IDictionary <string, string> telemetryProperties = new Dictionary <string, string> { { "Type", "Azure/EventHub" } }; object value; if (eventData.Properties.TryGetValue("Type", out value)) { telemetryProperties.Add("MessageType", value.ToString()); } TelemetryHelper.TrackMetric("Cqrs/Handle/Event", CurrentHandles++, telemetryProperties); // Do a manual 10 try attempt with back-off for (int i = 0; i < 10; i++) { try { Logger.LogDebug(string.Format("An event message arrived with the partition key '{0}', sequence number '{1}' and offset '{2}'.", eventData.PartitionKey, eventData.SequenceNumber, eventData.Offset)); string messageBody = Encoding.UTF8.GetString(eventData.GetBytes()); IEvent <TAuthenticationToken> @event = AzureBusHelper.ReceiveEvent(messageBody, ReceiveEvent, string.Format("partition key '{0}', sequence number '{1}' and offset '{2}'", eventData.PartitionKey, eventData.SequenceNumber, eventData.Offset), ExtractSignature(eventData), SigningTokenConfigurationKey, () => { wasSuccessfull = null; telemetryName = string.Format("Cqrs/Handle/Event/Skipped/{0}", eventData.SequenceNumber); responseCode = "204"; // Remove message from queue context.CheckpointAsync(eventData); Logger.LogDebug(string.Format("An event message arrived with the partition key '{0}', sequence number '{1}' and offset '{2}' but processing was skipped due to event settings.", eventData.PartitionKey, eventData.SequenceNumber, eventData.Offset)); TelemetryHelper.TrackEvent("Cqrs/Handle/Event/Skipped", telemetryProperties); } ); if (wasSuccessfull != null) { if (@event != null) { telemetryName = string.Format("{0}/{1}/{2}", @event.GetType().FullName, @event.GetIdentity(), @event.Id); authenticationToken = @event.AuthenticationToken as ISingleSignOnToken; if (AuthenticationTokenIsGuid) { guidAuthenticationToken = @event.AuthenticationToken as Guid?; } if (AuthenticationTokenIsString) { stringAuthenticationToken = @event.AuthenticationToken as string; } if (AuthenticationTokenIsInt) { intAuthenticationToken = @event.AuthenticationToken as int?; } var telemeteredMessage = @event as ITelemeteredMessage; if (telemeteredMessage != null) { telemetryName = telemeteredMessage.TelemetryName; } telemetryName = string.Format("Cqrs/Handle/Event/{0}", telemetryName); } // Remove message from queue context.CheckpointAsync(eventData); } Logger.LogDebug(string.Format("An event message arrived and was processed with the partition key '{0}', sequence number '{1}' and offset '{2}'.", eventData.PartitionKey, eventData.SequenceNumber, eventData.Offset)); IList <IEvent <TAuthenticationToken> > events; if (EventWaits.TryGetValue(@event.CorrelationId, out events)) { events.Add(@event); } wasSuccessfull = true; responseCode = "200"; return; } catch (UnAuthorisedMessageReceivedException exception) { TelemetryHelper.TrackException(exception, null, telemetryProperties); // Indicates a problem, unlock message in queue Logger.LogError(string.Format("An event message arrived with the partition key '{0}', sequence number '{1}' and offset '{2}' but was not authorised.", eventData.PartitionKey, eventData.SequenceNumber, eventData.Offset), exception: exception); wasSuccessfull = false; responseCode = "401"; telemetryProperties.Add("ExceptionType", exception.GetType().FullName); telemetryProperties.Add("ExceptionMessage", exception.Message); } catch (NoHandlersRegisteredException exception) { TelemetryHelper.TrackException(exception, null, telemetryProperties); // Indicates a problem, unlock message in queue Logger.LogError(string.Format("An event message arrived with the partition key '{0}', sequence number '{1}' and offset '{2}' but no handlers were found to process it.", eventData.PartitionKey, eventData.SequenceNumber, eventData.Offset), exception: exception); wasSuccessfull = false; responseCode = "501"; telemetryProperties.Add("ExceptionType", exception.GetType().FullName); telemetryProperties.Add("ExceptionMessage", exception.Message); } catch (NoHandlerRegisteredException exception) { TelemetryHelper.TrackException(exception, null, telemetryProperties); // Indicates a problem, unlock message in queue Logger.LogError(string.Format("An event message arrived with the partition key '{0}', sequence number '{1}' and offset '{2}'s but no handler was found to process it.", eventData.PartitionKey, eventData.SequenceNumber, eventData.Offset), exception: exception); wasSuccessfull = false; responseCode = "501"; telemetryProperties.Add("ExceptionType", exception.GetType().FullName); telemetryProperties.Add("ExceptionMessage", exception.Message); } catch (Exception exception) { // Indicates a problem, unlock message in queue Logger.LogError(string.Format("An event message arrived with the partition key '{0}', sequence number '{1}' and offset '{2}' but failed to be process.", eventData.PartitionKey, eventData.SequenceNumber, eventData.Offset), exception: exception); switch (i) { case 0: case 1: // 10 seconds Thread.Sleep(10 * 1000); break; case 2: case 3: // 30 seconds Thread.Sleep(30 * 1000); break; case 4: case 5: case 6: // 1 minute Thread.Sleep(60 * 1000); break; case 7: case 8: // 3 minutes Thread.Sleep(3 * 60 * 1000); break; case 9: telemetryProperties.Add("ExceptionType", exception.GetType().FullName); telemetryProperties.Add("ExceptionMessage", exception.Message); break; } wasSuccessfull = false; responseCode = "500"; } finally { // Eventually just accept it context.CheckpointAsync(eventData); TelemetryHelper.TrackMetric("Cqrs/Handle/Event", CurrentHandles--, telemetryProperties); mainStopWatch.Stop(); if (guidAuthenticationToken != null) { TelemetryHelper.TrackRequest ( telemetryName, guidAuthenticationToken, startedAt, mainStopWatch.Elapsed, responseCode, wasSuccessfull == null || wasSuccessfull.Value, telemetryProperties ); } else if (intAuthenticationToken != null) { TelemetryHelper.TrackRequest ( telemetryName, intAuthenticationToken, startedAt, mainStopWatch.Elapsed, responseCode, wasSuccessfull == null || wasSuccessfull.Value, telemetryProperties ); } else if (stringAuthenticationToken != null) { TelemetryHelper.TrackRequest ( telemetryName, stringAuthenticationToken, startedAt, mainStopWatch.Elapsed, responseCode, wasSuccessfull == null || wasSuccessfull.Value, telemetryProperties ); } else { TelemetryHelper.TrackRequest ( telemetryName, authenticationToken, startedAt, mainStopWatch.Elapsed, responseCode, wasSuccessfull == null || wasSuccessfull.Value, telemetryProperties ); } TelemetryHelper.Flush(); } } }
/// <summary> /// Using a <see cref="Task"/>, clears all dead letters from the topic and subscription of the /// provided <paramref name="topicName"/> and <paramref name="topicSubscriptionName"/>. /// </summary> /// <param name="topicName">The name of the topic.</param> /// <param name="topicSubscriptionName">The name of the subscription.</param> /// <returns></returns> protected virtual CancellationTokenSource CleanUpDeadLetters(string topicName, string topicSubscriptionName) { var brokeredMessageRenewCancellationTokenSource = new CancellationTokenSource(); IDictionary <string, string> telemetryProperties = new Dictionary <string, string> { { "Type", "Azure/Servicebus" } }; int lockIssues = 0; Action <BrokeredMessage, IMessage> leaveDeadlLetterInQueue = (deadLetterBrokeredMessage, deadLetterMessage) => { // Remove message from queue try { deadLetterBrokeredMessage.Abandon(); lockIssues = 0; } catch (MessageLockLostException) { lockIssues++; Logger.LogWarning(string.Format("The lock supplied for abandon for the skipped dead-letter message '{0}' is invalid.", deadLetterBrokeredMessage.MessageId)); } Logger.LogDebug(string.Format("A dead-letter message of type {0} arrived with the id '{1}' but left in the queue due to settings.", deadLetterMessage.GetType().FullName, deadLetterBrokeredMessage.MessageId)); }; Action <BrokeredMessage> removeDeadlLetterFromQueue = deadLetterBrokeredMessage => { // Remove message from queue try { deadLetterBrokeredMessage.Complete(); lockIssues = 0; } catch (MessageLockLostException) { lockIssues++; Logger.LogWarning(string.Format("The lock supplied for complete for the skipped dead-letter message '{0}' is invalid.", deadLetterBrokeredMessage.MessageId)); } Logger.LogDebug(string.Format("A dead-letter message arrived with the id '{0}' but was removed as processing was skipped due to settings.", deadLetterBrokeredMessage.MessageId)); }; Task.Factory.StartNewSafely(() => { int loop = 0; while (!brokeredMessageRenewCancellationTokenSource.Token.IsCancellationRequested) { lockIssues = 0; MessagingFactory factory = MessagingFactory.CreateFromConnectionString(ConnectionString); string deadLetterPath = SubscriptionClient.FormatDeadLetterPath(topicName, topicSubscriptionName); MessageReceiver client = factory.CreateMessageReceiver(deadLetterPath, ReceiveMode.PeekLock); IEnumerable <BrokeredMessage> brokeredMessages = client.ReceiveBatch(1000); foreach (BrokeredMessage brokeredMessage in brokeredMessages) { if (lockIssues > 10) { break; } try { Logger.LogDebug(string.Format("A dead-letter message arrived with the id '{0}'.", brokeredMessage.MessageId)); string messageBody = brokeredMessage.GetBody <string>(); // Closure protection BrokeredMessage message = brokeredMessage; try { AzureBusHelper.ReceiveEvent ( messageBody, @event => { bool isRequired = BusHelper.IsEventRequired(@event.GetType()); if (!isRequired) { removeDeadlLetterFromQueue(message); } else { leaveDeadlLetterInQueue(message, @event); } return(true); }, string.Format("id '{0}'", brokeredMessage.MessageId), () => { removeDeadlLetterFromQueue(message); }, () => { } ); } catch { AzureBusHelper.ReceiveCommand ( messageBody, command => { bool isRequired = BusHelper.IsEventRequired(command.GetType()); if (!isRequired) { removeDeadlLetterFromQueue(message); } else { leaveDeadlLetterInQueue(message, command); } return(true); }, string.Format("id '{0}'", brokeredMessage.MessageId), () => { removeDeadlLetterFromQueue(message); }, () => { } ); } } catch (Exception exception) { TelemetryHelper.TrackException(exception, null, telemetryProperties); // Indicates a problem, unlock message in queue Logger.LogError(string.Format("A dead-letter message arrived with the id '{0}' but failed to be process.", brokeredMessage.MessageId), exception: exception); try { brokeredMessage.Abandon(); } catch (MessageLockLostException) { lockIssues++; Logger.LogWarning(string.Format("The lock supplied for abandon for the skipped dead-letter message '{0}' is invalid.", brokeredMessage.MessageId)); } } } client.Close(); if (loop++ % 5 == 0) { loop = 0; Thread.Yield(); } else { Thread.Sleep(500); } } try { brokeredMessageRenewCancellationTokenSource.Dispose(); } catch (ObjectDisposedException) { } }, brokeredMessageRenewCancellationTokenSource.Token); return(brokeredMessageRenewCancellationTokenSource); }
protected virtual void ReceiveEvent(IMessageReceiver client, BrokeredMessage message) #endif { DateTimeOffset startedAt = DateTimeOffset.UtcNow; Stopwatch mainStopWatch = Stopwatch.StartNew(); string responseCode = "200"; // Null means it was skipped bool? wasSuccessfull = true; string telemetryName = string.Format("Cqrs/Handle/Event/{0}", message.MessageId); ISingleSignOnToken authenticationToken = null; Guid? guidAuthenticationToken = null; string stringAuthenticationToken = null; int?intAuthenticationToken = null; IDictionary <string, string> telemetryProperties = ExtractTelemetryProperties(message, "Azure/Servicebus"); TelemetryHelper.TrackMetric("Cqrs/Handle/Event", CurrentHandles++, telemetryProperties); var brokeredMessageRenewCancellationTokenSource = new CancellationTokenSource(); try { try { Logger.LogDebug(string.Format("An event message arrived with the id '{0}'.", message.MessageId)); string messageBody = message.GetBodyAsString(); IEvent <TAuthenticationToken> @event = AzureBusHelper.ReceiveEvent(messageBody, ReceiveEvent, string.Format("id '{0}'", message.MessageId), ExtractSignature(message), SigningTokenConfigurationKey, () => { wasSuccessfull = null; telemetryName = string.Format("Cqrs/Handle/Event/Skipped/{0}", message.MessageId); responseCode = "204"; // Remove message from queue try { #if NET452 message.Complete(); #endif #if NETSTANDARD2_0 client.CompleteAsync(message.SystemProperties.LockToken).Wait(1500); #endif } catch (AggregateException aggregateException) { if (aggregateException.InnerException is MessageLockLostException) { throw new MessageLockLostException(string.Format("The lock supplied for the skipped event message '{0}' is invalid.", message.MessageId), aggregateException.InnerException); } else { throw; } } catch (MessageLockLostException exception) { throw new MessageLockLostException(string.Format("The lock supplied for the skipped event message '{0}' is invalid.", message.MessageId), exception); } Logger.LogDebug(string.Format("An event message arrived with the id '{0}' but processing was skipped due to event settings.", message.MessageId)); TelemetryHelper.TrackEvent("Cqrs/Handle/Event/Skipped", telemetryProperties); }, () => { #if NET452 AzureBusHelper.RefreshLock(brokeredMessageRenewCancellationTokenSource, message, "event"); #endif #if NETSTANDARD2_0 AzureBusHelper.RefreshLock(client, brokeredMessageRenewCancellationTokenSource, message, "event"); #endif } ); if (wasSuccessfull != null) { if (@event != null) { telemetryName = string.Format("{0}/{1}/{2}", @event.GetType().FullName, @event.GetIdentity(), @event.Id); authenticationToken = @event.AuthenticationToken as ISingleSignOnToken; if (AuthenticationTokenIsGuid) { guidAuthenticationToken = @event.AuthenticationToken as Guid?; } if (AuthenticationTokenIsString) { stringAuthenticationToken = @event.AuthenticationToken as string; } if (AuthenticationTokenIsInt) { intAuthenticationToken = @event.AuthenticationToken as int?; } var telemeteredMessage = @event as ITelemeteredMessage; if (telemeteredMessage != null) { telemetryName = telemeteredMessage.TelemetryName; } telemetryName = string.Format("Cqrs/Handle/Event/{0}", telemetryName); } // Remove message from queue try { #if NET452 message.Complete(); #endif #if NETSTANDARD2_0 client.CompleteAsync(message.SystemProperties.LockToken).Wait(1500); #endif } catch (AggregateException aggregateException) { if (aggregateException.InnerException is MessageLockLostException) { throw new MessageLockLostException(string.Format("The lock supplied for event '{0}' of type {1} is invalid.", @event.Id, @event.GetType().Name), aggregateException.InnerException); } else { throw; } } catch (MessageLockLostException exception) { throw new MessageLockLostException(string.Format("The lock supplied for event '{0}' of type {1} is invalid.", @event.Id, @event.GetType().Name), exception); } } Logger.LogDebug(string.Format("An event message arrived and was processed with the id '{0}'.", message.MessageId)); IList <IEvent <TAuthenticationToken> > events; if (EventWaits.TryGetValue(@event.CorrelationId, out events)) { events.Add(@event); } } catch (AggregateException aggregateException) { throw aggregateException.InnerException; } } catch (MessageLockLostException exception) { IDictionary <string, string> subTelemetryProperties = new Dictionary <string, string>(telemetryProperties); subTelemetryProperties.Add("TimeTaken", mainStopWatch.Elapsed.ToString()); TelemetryHelper.TrackException(exception, null, subTelemetryProperties); if (ThrowExceptionOnReceiverMessageLockLostExceptionDuringComplete) { Logger.LogError(exception.Message, exception: exception); // Indicates a problem, unlock message in queue #if NET452 message.Abandon(); #endif #if NETSTANDARD2_0 client.AbandonAsync(message.SystemProperties.LockToken).Wait(1500); #endif wasSuccessfull = false; } else { Logger.LogWarning(exception.Message, exception: exception); try { #if NET452 message.DeadLetter("LockLostButHandled", "The message was handled but the lock was lost."); #endif #if NETSTANDARD2_0 client.DeadLetterAsync(message.SystemProperties.LockToken, "LockLostButHandled", "The message was handled but the lock was lost.").Wait(1500); #endif } catch (Exception) { // Oh well, move on. #if NET452 message.Abandon(); #endif #if NETSTANDARD2_0 client.AbandonAsync(message.SystemProperties.LockToken).Wait(1500); #endif } } responseCode = "599"; } catch (UnAuthorisedMessageReceivedException exception) { TelemetryHelper.TrackException(exception, null, telemetryProperties); // Indicates a problem, unlock message in queue Logger.LogError(string.Format("An event message arrived with the id '{0}' but was not authorised.", message.MessageId), exception: exception); #if NET452 message.DeadLetter("UnAuthorisedMessageReceivedException", exception.Message); #endif #if NETSTANDARD2_0 client.DeadLetterAsync(message.SystemProperties.LockToken, "UnAuthorisedMessageReceivedException", exception.Message).Wait(1500); #endif wasSuccessfull = false; responseCode = "401"; telemetryProperties.Add("ExceptionType", exception.GetType().FullName); telemetryProperties.Add("ExceptionMessage", exception.Message); } catch (NoHandlersRegisteredException exception) { TelemetryHelper.TrackException(exception, null, telemetryProperties); // Indicates a problem, unlock message in queue Logger.LogError(string.Format("An event message arrived with the id '{0}' but no handlers were found to process it.", message.MessageId), exception: exception); #if NET452 message.DeadLetter("NoHandlersRegisteredException", exception.Message); #endif #if NETSTANDARD2_0 client.DeadLetterAsync(message.SystemProperties.LockToken, "NoHandlersRegisteredException", exception.Message).Wait(1500); #endif wasSuccessfull = false; responseCode = "501"; telemetryProperties.Add("ExceptionType", exception.GetType().FullName); telemetryProperties.Add("ExceptionMessage", exception.Message); } catch (NoHandlerRegisteredException exception) { TelemetryHelper.TrackException(exception, null, telemetryProperties); // Indicates a problem, unlock message in queue Logger.LogError(string.Format("An event message arrived with the id '{0}' but no handler was found to process it.", message.MessageId), exception: exception); #if NET452 message.DeadLetter("NoHandlerRegisteredException", exception.Message); #endif #if NETSTANDARD2_0 client.DeadLetterAsync(message.SystemProperties.LockToken, "NoHandlerRegisteredException", exception.Message).Wait(1500); #endif wasSuccessfull = false; responseCode = "501"; telemetryProperties.Add("ExceptionType", exception.GetType().FullName); telemetryProperties.Add("ExceptionMessage", exception.Message); } catch (Exception exception) { TelemetryHelper.TrackException(exception, null, telemetryProperties); // Indicates a problem, unlock message in queue Logger.LogError(string.Format("An event message arrived with the id '{0}' but failed to be process.", message.MessageId), exception: exception); #if NET452 message.Abandon(); #endif #if NETSTANDARD2_0 client.AbandonAsync(message.SystemProperties.LockToken).Wait(1500); #endif wasSuccessfull = false; responseCode = "500"; telemetryProperties.Add("ExceptionType", exception.GetType().FullName); telemetryProperties.Add("ExceptionMessage", exception.Message); } finally { // Cancel the lock of renewing the task brokeredMessageRenewCancellationTokenSource.Cancel(); TelemetryHelper.TrackMetric("Cqrs/Handle/Event", CurrentHandles--, telemetryProperties); mainStopWatch.Stop(); if (guidAuthenticationToken != null) { TelemetryHelper.TrackRequest ( telemetryName, guidAuthenticationToken, startedAt, mainStopWatch.Elapsed, responseCode, wasSuccessfull == null || wasSuccessfull.Value, telemetryProperties ); } else if (intAuthenticationToken != null) { TelemetryHelper.TrackRequest ( telemetryName, intAuthenticationToken, startedAt, mainStopWatch.Elapsed, responseCode, wasSuccessfull == null || wasSuccessfull.Value, telemetryProperties ); } else if (stringAuthenticationToken != null) { TelemetryHelper.TrackRequest ( telemetryName, stringAuthenticationToken, startedAt, mainStopWatch.Elapsed, responseCode, wasSuccessfull == null || wasSuccessfull.Value, telemetryProperties ); } else { TelemetryHelper.TrackRequest ( telemetryName, authenticationToken, startedAt, mainStopWatch.Elapsed, responseCode, wasSuccessfull == null || wasSuccessfull.Value, telemetryProperties ); } TelemetryHelper.Flush(); } }
protected virtual void ReceiveEvent(BrokeredMessage message) { DateTimeOffset startedAt = DateTimeOffset.UtcNow; Stopwatch mainStopWatch = Stopwatch.StartNew(); string responseCode = "200"; // Null means it was skipped bool? wasSuccessfull = true; string telemetryName = string.Format("Cqrs/Handle/Event/{0}", message.MessageId); ISingleSignOnToken authenticationToken = null; IDictionary <string, string> telemetryProperties = new Dictionary <string, string> { { "Type", "Azure/Servicebus" } }; object value; if (message.Properties.TryGetValue("Type", out value)) { telemetryProperties.Add("MessageType", value.ToString()); } TelemetryHelper.TrackMetric("Cqrs/Handle/Event", CurrentHandles++, telemetryProperties); var brokeredMessageRenewCancellationTokenSource = new CancellationTokenSource(); try { Logger.LogDebug(string.Format("An event message arrived with the id '{0}'.", message.MessageId)); string messageBody = message.GetBody <string>(); IEvent <TAuthenticationToken> @event = AzureBusHelper.ReceiveEvent(messageBody, ReceiveEvent, string.Format("id '{0}'", message.MessageId), () => { wasSuccessfull = null; telemetryName = string.Format("Cqrs/Handle/Event/Skipped/{0}", message.MessageId); responseCode = "204"; // Remove message from queue try { message.Complete(); } catch (MessageLockLostException exception) { throw new MessageLockLostException(string.Format("The lock supplied for the skipped event message '{0}' is invalid.", message.MessageId), exception); } Logger.LogDebug(string.Format("An event message arrived with the id '{0}' but processing was skipped due to event settings.", message.MessageId)); TelemetryHelper.TrackEvent("Cqrs/Handle/Event/Skipped", telemetryProperties); }, () => { AzureBusHelper.RefreshLock(brokeredMessageRenewCancellationTokenSource, message, "event"); } ); if (wasSuccessfull != null) { if (@event != null) { telemetryName = string.Format("{0}/{1}", @event.GetType().FullName, @event.Id); authenticationToken = @event.AuthenticationToken as ISingleSignOnToken; var telemeteredMessage = @event as ITelemeteredMessage; if (telemeteredMessage != null) { telemetryName = telemeteredMessage.TelemetryName; } telemetryName = string.Format("Cqrs/Handle/Event/{0}", telemetryName); } // Remove message from queue try { message.Complete(); } catch (MessageLockLostException exception) { throw new MessageLockLostException(string.Format("The lock supplied for event '{0}' of type {1} is invalid.", @event.Id, @event.GetType().Name), exception); } } Logger.LogDebug(string.Format("An event message arrived and was processed with the id '{0}'.", message.MessageId)); IList <IEvent <TAuthenticationToken> > events; if (EventWaits.TryGetValue(@event.CorrelationId, out events)) { events.Add(@event); } } catch (Exception exception) { TelemetryHelper.TrackException(exception, null, telemetryProperties); // Indicates a problem, unlock message in queue Logger.LogError(string.Format("An event message arrived with the id '{0}' but failed to be process.", message.MessageId), exception: exception); message.Abandon(); wasSuccessfull = false; responseCode = "500"; } finally { // Cancel the lock of renewing the task brokeredMessageRenewCancellationTokenSource.Cancel(); TelemetryHelper.TrackMetric("Cqrs/Handle/Event", CurrentHandles--, telemetryProperties); mainStopWatch.Stop(); TelemetryHelper.TrackRequest ( telemetryName, authenticationToken, startedAt, mainStopWatch.Elapsed, responseCode, wasSuccessfull == null || wasSuccessfull.Value, telemetryProperties ); TelemetryHelper.Flush(); } }
/// <summary> /// Using a <see cref="Task"/>, clears all dead letters from the topic and subscription of the /// provided <paramref name="topicName"/> and <paramref name="topicSubscriptionName"/>. /// </summary> /// <param name="topicName">The name of the topic.</param> /// <param name="topicSubscriptionName">The name of the subscription.</param> /// <returns></returns> protected virtual CancellationTokenSource CleanUpDeadLetters(string topicName, string topicSubscriptionName) { var brokeredMessageRenewCancellationTokenSource = new CancellationTokenSource(); IDictionary <string, string> telemetryProperties = new Dictionary <string, string> { { "Type", "Azure/Servicebus" } }; int lockIssues = 0; #if NET452 Action <BrokeredMessage, IMessage> leaveDeadlLetterInQueue = (deadLetterBrokeredMessage, deadLetterMessage) => #endif #if NETSTANDARD2_0 Action <IMessageReceiver, BrokeredMessage, IMessage> leaveDeadlLetterInQueue = (client, deadLetterBrokeredMessage, deadLetterMessage) => #endif { // Remove message from queue try { #if NET452 deadLetterBrokeredMessage.Abandon(); #endif #if NETSTANDARD2_0 client.AbandonAsync(deadLetterBrokeredMessage.SystemProperties.LockToken).Wait(1500); #endif lockIssues = 0; } catch (MessageLockLostException) { lockIssues++; Logger.LogWarning(string.Format("The lock supplied for abandon for the skipped dead-letter message '{0}' is invalid.", deadLetterBrokeredMessage.MessageId)); } Logger.LogDebug(string.Format("A dead-letter message of type {0} arrived with the id '{1}' but left in the queue due to settings.", deadLetterMessage.GetType().FullName, deadLetterBrokeredMessage.MessageId)); }; #if NET452 Action <BrokeredMessage> removeDeadlLetterFromQueue = (deadLetterBrokeredMessage) => #endif #if NETSTANDARD2_0 Action <IMessageReceiver, BrokeredMessage> removeDeadlLetterFromQueue = (client, deadLetterBrokeredMessage) => #endif { // Remove message from queue try { #if NET452 deadLetterBrokeredMessage.Complete(); #endif #if NETSTANDARD2_0 client.CompleteAsync(deadLetterBrokeredMessage.SystemProperties.LockToken).Wait(1500); #endif lockIssues = 0; } catch (MessageLockLostException) { lockIssues++; Logger.LogWarning(string.Format("The lock supplied for complete for the skipped dead-letter message '{0}' is invalid.", deadLetterBrokeredMessage.MessageId)); } Logger.LogDebug(string.Format("A dead-letter message arrived with the id '{0}' but was removed as processing was skipped due to settings.", deadLetterBrokeredMessage.MessageId)); }; Task.Factory.StartNewSafely(() => { int loop = 0; while (!brokeredMessageRenewCancellationTokenSource.Token.IsCancellationRequested) { lockIssues = 0; IEnumerable <BrokeredMessage> brokeredMessages; #if NET452 MessagingFactory factory = MessagingFactory.CreateFromConnectionString(ConnectionString); string deadLetterPath = SubscriptionClient.FormatDeadLetterPath(topicName, topicSubscriptionName); MessageReceiver client = factory.CreateMessageReceiver(deadLetterPath, ReceiveMode.PeekLock); brokeredMessages = client.ReceiveBatch(1000); #endif #if NETSTANDARD2_0 string deadLetterPath = EntityNameHelper.FormatDeadLetterPath(EntityNameHelper.FormatSubscriptionPath(topicName, topicSubscriptionName)); MessageReceiver client = new MessageReceiver(ConnectionString, deadLetterPath, ReceiveMode.PeekLock); Task <IList <BrokeredMessage> > receiveTask = client.ReceiveAsync(1000); receiveTask.Wait(10000); if (receiveTask.IsCompleted && receiveTask.Result != null) { brokeredMessages = receiveTask.Result; } else { brokeredMessages = Enumerable.Empty <BrokeredMessage>(); } #endif foreach (BrokeredMessage brokeredMessage in brokeredMessages) { if (lockIssues > 10) { break; } try { Logger.LogDebug(string.Format("A dead-letter message arrived with the id '{0}'.", brokeredMessage.MessageId)); string messageBody = brokeredMessage.GetBodyAsString(); // Closure protection BrokeredMessage message = brokeredMessage; try { AzureBusHelper.ReceiveEvent ( messageBody, @event => { bool isRequired = BusHelper.IsEventRequired(@event.GetType()); if (!isRequired) { #if NET452 removeDeadlLetterFromQueue(message); #endif #if NETSTANDARD2_0 removeDeadlLetterFromQueue(client, message); #endif } else { #if NET452 leaveDeadlLetterInQueue(message, @event); #endif #if NETSTANDARD2_0 leaveDeadlLetterInQueue(client, message, @event); #endif } return(true); }, string.Format("id '{0}'", brokeredMessage.MessageId), ExtractSignature(message), SigningTokenConfigurationKey, () => { #if NET452 removeDeadlLetterFromQueue(message); #endif #if NETSTANDARD2_0 removeDeadlLetterFromQueue(client, message); #endif }, () => { } ); } catch { AzureBusHelper.ReceiveCommand ( messageBody, command => { bool isRequired = BusHelper.IsEventRequired(command.GetType()); if (!isRequired) { #if NET452 removeDeadlLetterFromQueue(message); #endif #if NETSTANDARD2_0 removeDeadlLetterFromQueue(client, message); #endif } else { #if NET452 leaveDeadlLetterInQueue(message, command); #endif #if NETSTANDARD2_0 leaveDeadlLetterInQueue(client, message, command); #endif } return(true); }, string.Format("id '{0}'", brokeredMessage.MessageId), ExtractSignature(message), SigningTokenConfigurationKey, () => { #if NET452 removeDeadlLetterFromQueue(message); #endif #if NETSTANDARD2_0 removeDeadlLetterFromQueue(client, message); #endif }, () => { } ); } } catch (Exception exception) { TelemetryHelper.TrackException(exception, null, telemetryProperties); // Indicates a problem, unlock message in queue Logger.LogError(string.Format("A dead-letter message arrived with the id '{0}' but failed to be process.", brokeredMessage.MessageId), exception: exception); try { #if NET452 brokeredMessage.Abandon(); #endif #if NETSTANDARD2_0 client.AbandonAsync(brokeredMessage.SystemProperties.LockToken).Wait(1500); #endif } catch (MessageLockLostException) { lockIssues++; Logger.LogWarning(string.Format("The lock supplied for abandon for the skipped dead-letter message '{0}' is invalid.", brokeredMessage.MessageId)); } } } #if NET452 client.Close(); #endif #if NETSTANDARD2_0 client.CloseAsync().Wait(1500); #endif if (loop++ % 5 == 0) { loop = 0; Thread.Yield(); } else { Thread.Sleep(500); } } try { brokeredMessageRenewCancellationTokenSource.Dispose(); } catch (ObjectDisposedException) { } }, brokeredMessageRenewCancellationTokenSource.Token); return(brokeredMessageRenewCancellationTokenSource); }
protected virtual void ReceiveEvent(PartitionContext context, EventData eventData) { IDictionary <string, string> telemetryProperties = new Dictionary <string, string> { { "Type", "Azure/EventHub" } }; TelemetryHelper.TrackMetric("Cqrs/Handle/Event", CurrentHandles++, telemetryProperties); // Do a manual 10 try attempt with back-off for (int i = 0; i < 10; i++) { try { Logger.LogDebug(string.Format("An event message arrived with the partition key '{0}', sequence number '{1}' and offset '{2}'.", eventData.PartitionKey, eventData.SequenceNumber, eventData.Offset)); string messageBody = Encoding.UTF8.GetString(eventData.GetBytes()); IEvent <TAuthenticationToken> @event = AzureBusHelper.ReceiveEvent(messageBody, ReceiveEvent, string.Format("partition key '{0}', sequence number '{1}' and offset '{2}'", eventData.PartitionKey, eventData.SequenceNumber, eventData.Offset), () => { // Remove message from queue context.CheckpointAsync(eventData); Logger.LogDebug(string.Format("An event message arrived with the partition key '{0}', sequence number '{1}' and offset '{2}' but processing was skipped due to event settings.", eventData.PartitionKey, eventData.SequenceNumber, eventData.Offset)); } ); // Remove message from queue context.CheckpointAsync(eventData); Logger.LogDebug(string.Format("An event message arrived and was processed with the partition key '{0}', sequence number '{1}' and offset '{2}'.", eventData.PartitionKey, eventData.SequenceNumber, eventData.Offset)); IList <IEvent <TAuthenticationToken> > events; if (EventWaits.TryGetValue(@event.CorrelationId, out events)) { events.Add(@event); } return; } catch (Exception exception) { // Indicates a problem, unlock message in queue Logger.LogError(string.Format("An event message arrived with the partition key '{0}', sequence number '{1}' and offset '{2}' but failed to be process.", eventData.PartitionKey, eventData.SequenceNumber, eventData.Offset), exception: exception); switch (i) { case 0: case 1: // 10 seconds Thread.Sleep(10 * 1000); break; case 2: case 3: // 30 seconds Thread.Sleep(30 * 1000); break; case 4: case 5: case 6: // 1 minute Thread.Sleep(60 * 1000); break; case 7: case 8: case 9: // 3 minutes Thread.Sleep(3 * 60 * 1000); break; } } } // Eventually just accept it context.CheckpointAsync(eventData); TelemetryHelper.TrackMetric("Cqrs/Handle/Event", CurrentHandles--, telemetryProperties); }