protected virtual void ReceiveCommand(BrokeredMessage message)
{
var brokeredMessageRenewCancellationTokenSource = new CancellationTokenSource();
try
{
Logger.LogDebug(string.Format("A command message arrived with the id '{0}'.", message.MessageId));
string messageBody = message.GetBody <string>();
AzureBusHelper.ReceiveCommand(messageBody, ReceiveCommand,
string.Format("id '{0}'", message.MessageId),
() =>
{
// Remove message from queue
message.Complete();
Logger.LogDebug(string.Format("A command message arrived with the id '{0}' but processing was skipped due to command settings.", message.MessageId));
},
() =>
{
Task.Factory.StartNew(() =>
{
while (!brokeredMessageRenewCancellationTokenSource.Token.IsCancellationRequested)
{
//Based on LockedUntilUtc property to determine if the lock expires soon
if (DateTime.UtcNow > message.LockedUntilUtc.AddSeconds(-10))
{
// If so, repeat the message
message.RenewLock();
}
Thread.Sleep(500);
}
}, brokeredMessageRenewCancellationTokenSource.Token);
}
);
// Remove message from queue
message.Complete();
Logger.LogDebug(string.Format("A command message arrived and was processed with the id '{0}'.", message.MessageId));
}
catch (Exception exception)
{
// Indicates a problem, unlock message in queue
Logger.LogError(string.Format("A command message arrived with the id '{0}' but failed to be process.", message.MessageId), exception: exception);
message.Abandon();
}
finally
{
// Cancel the lock of renewing the task
brokeredMessageRenewCancellationTokenSource.Cancel();
}
}
protected virtual void ReceiveCommand(BrokeredMessage message)
{
DateTimeOffset startedAt = DateTimeOffset.UtcNow;
Stopwatch mainStopWatch = Stopwatch.StartNew();
string responseCode = "200";
// Null means it was skipped
bool?wasSuccessfull = true;
IDictionary <string, string> telemetryProperties = new Dictionary <string, string> {
{ "Type", "Azure/Servicebus" }
};
object value;
if (message.Properties.TryGetValue("Type", out value))
{
telemetryProperties.Add("MessageType", value.ToString());
}
TelemetryHelper.TrackMetric("Cqrs/Handle/Command", CurrentHandles++, telemetryProperties);
var brokeredMessageRenewCancellationTokenSource = new CancellationTokenSource();
try
{
Logger.LogDebug(string.Format("A command message arrived with the id '{0}'.", message.MessageId));
string messageBody = message.GetBody <string>();
ICommand <TAuthenticationToken> command = AzureBusHelper.ReceiveCommand(messageBody, ReceiveCommand,
string.Format("id '{0}'", message.MessageId),
() =>
{
wasSuccessfull = null;
// Remove message from queue
try
{
message.Complete();
}
catch (MessageLockLostException exception)
{
throw new MessageLockLostException(string.Format("The lock supplied for the skipped command message '{0}' is invalid.", message.MessageId), exception);
}
Logger.LogDebug(string.Format("A command message arrived with the id '{0}' but processing was skipped due to event settings.", message.MessageId));
TelemetryHelper.TrackEvent("Cqrs/Handle/Command/Skipped", telemetryProperties);
},
() =>
{
AzureBusHelper.RefreshLock(brokeredMessageRenewCancellationTokenSource, message, "command");
}
);
if (wasSuccessfull != null)
{
// Remove message from queue
try
{
message.Complete();
}
catch (MessageLockLostException exception)
{
throw new MessageLockLostException(string.Format("The lock supplied for command '{0}' of type {1} is invalid.", command.Id, command.GetType().Name), exception);
}
}
Logger.LogDebug(string.Format("A command message arrived and was processed with the id '{0}'.", message.MessageId));
}
catch (Exception exception)
{
TelemetryHelper.TrackException(exception, null, telemetryProperties);
// Indicates a problem, unlock message in queue
Logger.LogError(string.Format("A command message arrived with the id '{0}' but failed to be process.", message.MessageId), exception: exception);
message.Abandon();
wasSuccessfull = false;
}
finally
{
// Cancel the lock of renewing the task
brokeredMessageRenewCancellationTokenSource.Cancel();
TelemetryHelper.TrackMetric("Cqrs/Handle/Command", CurrentHandles--, telemetryProperties);
mainStopWatch.Stop();
if (wasSuccessfull == null || !wasSuccessfull.Value)
{
TelemetryHelper.TrackRequest
(
string.Format("Cqrs/Handle/Command/Skipped/{0}", message.MessageId),
startedAt,
mainStopWatch.Elapsed,
responseCode,
wasSuccessfull == null,
telemetryProperties
);
}
TelemetryHelper.Flush();
}
}
/// <summary>
/// Receives a <see cref="BrokeredMessage"/> from the command bus.
/// </summary>
protected virtual void ReceiveCommand(BrokeredMessage message)
{
DateTimeOffset startedAt = DateTimeOffset.UtcNow;
Stopwatch mainStopWatch = Stopwatch.StartNew();
string responseCode = "200";
// Null means it was skipped
bool? wasSuccessfull = true;
string telemetryName = string.Format("Cqrs/Handle/Command/{0}", message.MessageId);
ISingleSignOnToken authenticationToken = null;
Guid? guidAuthenticationToken = null;
string stringAuthenticationToken = null;
int?intAuthenticationToken = null;
IDictionary <string, string> telemetryProperties = ExtractTelemetryProperties(message, "Azure/Servicebus");
TelemetryHelper.TrackMetric("Cqrs/Handle/Command", CurrentHandles++, telemetryProperties);
var brokeredMessageRenewCancellationTokenSource = new CancellationTokenSource();
try
{
Logger.LogDebug(string.Format("A command message arrived with the id '{0}'.", message.MessageId));
string messageBody = message.GetBody <string>();
ICommand <TAuthenticationToken> command = AzureBusHelper.ReceiveCommand(messageBody, ReceiveCommand,
string.Format("id '{0}'", message.MessageId),
ExtractSignature(message),
SigningTokenConfigurationKey,
() =>
{
wasSuccessfull = null;
telemetryName = string.Format("Cqrs/Handle/Command/Skipped/{0}", message.MessageId);
responseCode = "204";
// Remove message from queue
try
{
message.Complete();
}
catch (MessageLockLostException exception)
{
throw new MessageLockLostException(string.Format("The lock supplied for the skipped command message '{0}' is invalid.", message.MessageId), exception);
}
Logger.LogDebug(string.Format("A command message arrived with the id '{0}' but processing was skipped due to event settings.", message.MessageId));
TelemetryHelper.TrackEvent("Cqrs/Handle/Command/Skipped", telemetryProperties);
},
() =>
{
AzureBusHelper.RefreshLock(brokeredMessageRenewCancellationTokenSource, message, "command");
}
);
if (wasSuccessfull != null)
{
if (command != null)
{
telemetryName = string.Format("{0}/{1}", command.GetType().FullName, command.Id);
authenticationToken = command.AuthenticationToken as ISingleSignOnToken;
if (AuthenticationTokenIsGuid)
{
guidAuthenticationToken = command.AuthenticationToken as Guid?;
}
if (AuthenticationTokenIsString)
{
stringAuthenticationToken = command.AuthenticationToken as string;
}
if (AuthenticationTokenIsInt)
{
intAuthenticationToken = command.AuthenticationToken as int?;
}
var telemeteredMessage = command as ITelemeteredMessage;
if (telemeteredMessage != null)
{
telemetryName = telemeteredMessage.TelemetryName;
}
telemetryName = string.Format("Cqrs/Handle/Command/{0}", telemetryName);
}
// Remove message from queue
try
{
message.Complete();
}
catch (MessageLockLostException exception)
{
throw new MessageLockLostException(string.Format("The lock supplied for command '{0}' of type {1} is invalid.", command.Id, command.GetType().Name), exception);
}
}
Logger.LogDebug(string.Format("A command message arrived and was processed with the id '{0}'.", message.MessageId));
}
catch (MessageLockLostException exception)
{
IDictionary <string, string> subTelemetryProperties = new Dictionary <string, string>(telemetryProperties);
subTelemetryProperties.Add("TimeTaken", mainStopWatch.Elapsed.ToString());
TelemetryHelper.TrackException(exception, null, subTelemetryProperties);
if (ThrowExceptionOnReceiverMessageLockLostExceptionDuringComplete)
{
Logger.LogError(exception.Message, exception: exception);
// Indicates a problem, unlock message in queue
message.Abandon();
wasSuccessfull = false;
}
else
{
Logger.LogWarning(exception.Message, exception: exception);
try
{
message.DeadLetter("LockLostButHandled", "The message was handled but the lock was lost.");
}
catch (Exception)
{
// Oh well, move on.
message.Abandon();
}
}
responseCode = "599";
}
catch (UnAuthorisedMessageReceivedException exception)
{
TelemetryHelper.TrackException(exception, null, telemetryProperties);
// Indicates a problem, unlock message in queue
Logger.LogError(string.Format("A command message arrived with the id '{0}' but was not authorised.", message.MessageId), exception: exception);
message.DeadLetter("UnAuthorisedMessageReceivedException", exception.Message);
wasSuccessfull = false;
responseCode = "401";
telemetryProperties.Add("ExceptionType", exception.GetType().FullName);
telemetryProperties.Add("ExceptionMessage", exception.Message);
}
catch (NoHandlersRegisteredException exception)
{
TelemetryHelper.TrackException(exception, null, telemetryProperties);
// Indicates a problem, unlock message in queue
Logger.LogError(string.Format("A command message arrived with the id '{0}' but no handlers were found to process it.", message.MessageId), exception: exception);
message.DeadLetter("NoHandlersRegisteredException", exception.Message);
wasSuccessfull = false;
responseCode = "501";
telemetryProperties.Add("ExceptionType", exception.GetType().FullName);
telemetryProperties.Add("ExceptionMessage", exception.Message);
}
catch (NoHandlerRegisteredException exception)
{
TelemetryHelper.TrackException(exception, null, telemetryProperties);
// Indicates a problem, unlock message in queue
Logger.LogError(string.Format("A command message arrived with the id '{0}' but no handler was found to process it.", message.MessageId), exception: exception);
message.DeadLetter("NoHandlerRegisteredException", exception.Message);
wasSuccessfull = false;
responseCode = "501";
telemetryProperties.Add("ExceptionType", exception.GetType().FullName);
telemetryProperties.Add("ExceptionMessage", exception.Message);
}
catch (Exception exception)
{
TelemetryHelper.TrackException(exception, null, telemetryProperties);
// Indicates a problem, unlock message in queue
Logger.LogError(string.Format("A command message arrived with the id '{0}' but failed to be process.", message.MessageId), exception: exception);
message.Abandon();
wasSuccessfull = false;
responseCode = "500";
telemetryProperties.Add("ExceptionType", exception.GetType().FullName);
telemetryProperties.Add("ExceptionMessage", exception.Message);
}
finally
{
// Cancel the lock of renewing the task
brokeredMessageRenewCancellationTokenSource.Cancel();
TelemetryHelper.TrackMetric("Cqrs/Handle/Command", CurrentHandles--, telemetryProperties);
mainStopWatch.Stop();
if (guidAuthenticationToken != null)
{
TelemetryHelper.TrackRequest
(
telemetryName,
guidAuthenticationToken,
startedAt,
mainStopWatch.Elapsed,
responseCode,
wasSuccessfull == null || wasSuccessfull.Value,
telemetryProperties
);
}
else if (intAuthenticationToken != null)
{
TelemetryHelper.TrackRequest
(
telemetryName,
intAuthenticationToken,
startedAt,
mainStopWatch.Elapsed,
responseCode,
wasSuccessfull == null || wasSuccessfull.Value,
telemetryProperties
);
}
else if (stringAuthenticationToken != null)
{
TelemetryHelper.TrackRequest
(
telemetryName,
stringAuthenticationToken,
startedAt,
mainStopWatch.Elapsed,
responseCode,
wasSuccessfull == null || wasSuccessfull.Value,
telemetryProperties
);
}
else
{
TelemetryHelper.TrackRequest
(
telemetryName,
authenticationToken,
startedAt,
mainStopWatch.Elapsed,
responseCode,
wasSuccessfull == null || wasSuccessfull.Value,
telemetryProperties
);
}
TelemetryHelper.Flush();
}
}
/// <summary>
/// Using a <see cref="Task"/>, clears all dead letters from the topic and subscription of the
/// provided <paramref name="topicName"/> and <paramref name="topicSubscriptionName"/>.
/// </summary>
/// <param name="topicName">The name of the topic.</param>
/// <param name="topicSubscriptionName">The name of the subscription.</param>
/// <returns></returns>
protected virtual CancellationTokenSource CleanUpDeadLetters(string topicName, string topicSubscriptionName)
{
var brokeredMessageRenewCancellationTokenSource = new CancellationTokenSource();
IDictionary <string, string> telemetryProperties = new Dictionary <string, string> {
{ "Type", "Azure/Servicebus" }
};
int lockIssues = 0;
Action <BrokeredMessage, IMessage> leaveDeadlLetterInQueue = (deadLetterBrokeredMessage, deadLetterMessage) =>
{
// Remove message from queue
try
{
deadLetterBrokeredMessage.Abandon();
lockIssues = 0;
}
catch (MessageLockLostException)
{
lockIssues++;
Logger.LogWarning(string.Format("The lock supplied for abandon for the skipped dead-letter message '{0}' is invalid.", deadLetterBrokeredMessage.MessageId));
}
Logger.LogDebug(string.Format("A dead-letter message of type {0} arrived with the id '{1}' but left in the queue due to settings.", deadLetterMessage.GetType().FullName, deadLetterBrokeredMessage.MessageId));
};
Action <BrokeredMessage> removeDeadlLetterFromQueue = deadLetterBrokeredMessage =>
{
// Remove message from queue
try
{
deadLetterBrokeredMessage.Complete();
lockIssues = 0;
}
catch (MessageLockLostException)
{
lockIssues++;
Logger.LogWarning(string.Format("The lock supplied for complete for the skipped dead-letter message '{0}' is invalid.", deadLetterBrokeredMessage.MessageId));
}
Logger.LogDebug(string.Format("A dead-letter message arrived with the id '{0}' but was removed as processing was skipped due to settings.", deadLetterBrokeredMessage.MessageId));
};
Task.Factory.StartNewSafely(() =>
{
int loop = 0;
while (!brokeredMessageRenewCancellationTokenSource.Token.IsCancellationRequested)
{
lockIssues = 0;
MessagingFactory factory = MessagingFactory.CreateFromConnectionString(ConnectionString);
string deadLetterPath = SubscriptionClient.FormatDeadLetterPath(topicName, topicSubscriptionName);
MessageReceiver client = factory.CreateMessageReceiver(deadLetterPath, ReceiveMode.PeekLock);
IEnumerable <BrokeredMessage> brokeredMessages = client.ReceiveBatch(1000);
foreach (BrokeredMessage brokeredMessage in brokeredMessages)
{
if (lockIssues > 10)
{
break;
}
try
{
Logger.LogDebug(string.Format("A dead-letter message arrived with the id '{0}'.", brokeredMessage.MessageId));
string messageBody = brokeredMessage.GetBody <string>();
// Closure protection
BrokeredMessage message = brokeredMessage;
try
{
AzureBusHelper.ReceiveEvent
(
messageBody,
@event =>
{
bool isRequired = BusHelper.IsEventRequired(@event.GetType());
if (!isRequired)
{
removeDeadlLetterFromQueue(message);
}
else
{
leaveDeadlLetterInQueue(message, @event);
}
return(true);
},
string.Format("id '{0}'", brokeredMessage.MessageId),
() =>
{
removeDeadlLetterFromQueue(message);
},
() => { }
);
}
catch
{
AzureBusHelper.ReceiveCommand
(
messageBody,
command =>
{
bool isRequired = BusHelper.IsEventRequired(command.GetType());
if (!isRequired)
{
removeDeadlLetterFromQueue(message);
}
else
{
leaveDeadlLetterInQueue(message, command);
}
return(true);
},
string.Format("id '{0}'", brokeredMessage.MessageId),
() =>
{
removeDeadlLetterFromQueue(message);
},
() => { }
);
}
}
catch (Exception exception)
{
TelemetryHelper.TrackException(exception, null, telemetryProperties);
// Indicates a problem, unlock message in queue
Logger.LogError(string.Format("A dead-letter message arrived with the id '{0}' but failed to be process.", brokeredMessage.MessageId), exception: exception);
try
{
brokeredMessage.Abandon();
}
catch (MessageLockLostException)
{
lockIssues++;
Logger.LogWarning(string.Format("The lock supplied for abandon for the skipped dead-letter message '{0}' is invalid.", brokeredMessage.MessageId));
}
}
}
client.Close();
if (loop++ % 5 == 0)
{
loop = 0;
Thread.Yield();
}
else
{
Thread.Sleep(500);
}
}
try
{
brokeredMessageRenewCancellationTokenSource.Dispose();
}
catch (ObjectDisposedException) { }
}, brokeredMessageRenewCancellationTokenSource.Token);
return(brokeredMessageRenewCancellationTokenSource);
}
protected virtual void ReceiveCommand(PartitionContext context, EventData eventData)
{
// Do a manual 10 try attempt with back-off
for (int i = 0; i < 10; i++)
{
try
{
Logger.LogDebug(string.Format("A command message arrived with the partition key '{0}', sequence number '{1}' and offset '{2}'.", eventData.PartitionKey, eventData.SequenceNumber, eventData.Offset));
string messageBody = Encoding.UTF8.GetString(eventData.GetBytes());
AzureBusHelper.ReceiveCommand(messageBody, ReceiveCommand,
string.Format("partition key '{0}', sequence number '{1}' and offset '{2}'", eventData.PartitionKey, eventData.SequenceNumber, eventData.Offset),
() =>
{
// Remove message from queue
context.CheckpointAsync(eventData);
Logger.LogDebug(string.Format("A command message arrived with the partition key '{0}', sequence number '{1}' and offset '{2}' but processing was skipped due to command settings.", eventData.PartitionKey, eventData.SequenceNumber, eventData.Offset));
}
);
// Remove message from queue
context.CheckpointAsync(eventData);
Logger.LogDebug(string.Format("A command message arrived and was processed with the partition key '{0}', sequence number '{1}' and offset '{2}'.", eventData.PartitionKey, eventData.SequenceNumber, eventData.Offset));
return;
}
catch (Exception exception)
{
// Indicates a problem, unlock message in queue
Logger.LogError(string.Format("A command message arrived with the partition key '{0}', sequence number '{1}' and offset '{2}' but failed to be process.", eventData.PartitionKey, eventData.SequenceNumber, eventData.Offset), exception: exception);
switch (i)
{
case 0:
case 1:
// 10 seconds
Thread.Sleep(10 * 1000);
break;
case 2:
case 3:
// 30 seconds
Thread.Sleep(30 * 1000);
break;
case 4:
case 5:
case 6:
// 1 minute
Thread.Sleep(60 * 1000);
break;
case 7:
case 8:
case 9:
// 3 minutes
Thread.Sleep(3 * 60 * 1000);
break;
}
}
}
// Eventually just accept it
context.CheckpointAsync(eventData);
}
/// <summary>
/// Receives <see cref="EventData"/> from the command bus.
/// </summary>
protected virtual void ReceiveCommand(PartitionContext context, EventData eventData)
{
DateTimeOffset startedAt = DateTimeOffset.UtcNow;
Stopwatch mainStopWatch = Stopwatch.StartNew();
string responseCode = "200";
// Null means it was skipped
bool? wasSuccessfull = true;
string telemetryName = string.Format("Cqrs/Handle/Command/{0}", eventData.SequenceNumber);
ISingleSignOnToken authenticationToken = null;
IDictionary <string, string> telemetryProperties = new Dictionary <string, string> {
{ "Type", "Azure/EventHub" }
};
object value;
if (eventData.Properties.TryGetValue("Type", out value))
{
telemetryProperties.Add("MessageType", value.ToString());
}
TelemetryHelper.TrackMetric("Cqrs/Handle/Command", CurrentHandles++, telemetryProperties);
// Do a manual 10 try attempt with back-off
for (int i = 0; i < 10; i++)
{
try
{
Logger.LogDebug(string.Format("A command message arrived with the partition key '{0}', sequence number '{1}' and offset '{2}'.", eventData.PartitionKey, eventData.SequenceNumber, eventData.Offset));
string messageBody = Encoding.UTF8.GetString(eventData.GetBytes());
ICommand <TAuthenticationToken> command = AzureBusHelper.ReceiveCommand(messageBody, ReceiveCommand,
string.Format("partition key '{0}', sequence number '{1}' and offset '{2}'", eventData.PartitionKey, eventData.SequenceNumber, eventData.Offset),
() =>
{
wasSuccessfull = null;
telemetryName = string.Format("Cqrs/Handle/Command/Skipped/{0}", eventData.SequenceNumber);
responseCode = "204";
// Remove message from queue
context.CheckpointAsync(eventData);
Logger.LogDebug(string.Format("A command message arrived with the partition key '{0}', sequence number '{1}' and offset '{2}' but processing was skipped due to command settings.", eventData.PartitionKey, eventData.SequenceNumber, eventData.Offset));
TelemetryHelper.TrackEvent("Cqrs/Handle/Command/Skipped", telemetryProperties);
}
);
if (wasSuccessfull != null)
{
if (command != null)
{
telemetryName = string.Format("{0}/{1}", command.GetType().FullName, command.Id);
authenticationToken = command.AuthenticationToken as ISingleSignOnToken;
var telemeteredMessage = command as ITelemeteredMessage;
if (telemeteredMessage != null)
{
telemetryName = telemeteredMessage.TelemetryName;
}
telemetryName = string.Format("Cqrs/Handle/Command/{0}", telemetryName);
}
// Remove message from queue
context.CheckpointAsync(eventData);
}
Logger.LogDebug(string.Format("A command message arrived and was processed with the partition key '{0}', sequence number '{1}' and offset '{2}'.", eventData.PartitionKey, eventData.SequenceNumber, eventData.Offset));
wasSuccessfull = true;
responseCode = "200";
return;
}
catch (Exception exception)
{
// Indicates a problem, unlock message in queue
Logger.LogError(string.Format("A command message arrived with the partition key '{0}', sequence number '{1}' and offset '{2}' but failed to be process.", eventData.PartitionKey, eventData.SequenceNumber, eventData.Offset), exception: exception);
switch (i)
{
case 0:
case 1:
// 10 seconds
Thread.Sleep(10 * 1000);
break;
case 2:
case 3:
// 30 seconds
Thread.Sleep(30 * 1000);
break;
case 4:
case 5:
case 6:
// 1 minute
Thread.Sleep(60 * 1000);
break;
case 7:
case 8:
// 3 minutes
Thread.Sleep(3 * 60 * 1000);
break;
case 9:
telemetryProperties.Add("ExceptionType", exception.GetType().FullName);
telemetryProperties.Add("ExceptionMessage", exception.Message);
break;
}
wasSuccessfull = false;
responseCode = "500";
}
finally
{
// Eventually just accept it
context.CheckpointAsync(eventData);
TelemetryHelper.TrackMetric("Cqrs/Handle/Command", CurrentHandles--, telemetryProperties);
mainStopWatch.Stop();
TelemetryHelper.TrackRequest
(
telemetryName,
authenticationToken,
startedAt,
mainStopWatch.Elapsed,
responseCode,
wasSuccessfull == null || wasSuccessfull.Value,
telemetryProperties
);
TelemetryHelper.Flush();
}
}
}
/// <summary>
/// Using a <see cref="Task"/>, clears all dead letters from the topic and subscription of the
/// provided <paramref name="topicName"/> and <paramref name="topicSubscriptionName"/>.
/// </summary>
/// <param name="topicName">The name of the topic.</param>
/// <param name="topicSubscriptionName">The name of the subscription.</param>
/// <returns></returns>
protected virtual CancellationTokenSource CleanUpDeadLetters(string topicName, string topicSubscriptionName)
{
var brokeredMessageRenewCancellationTokenSource = new CancellationTokenSource();
IDictionary <string, string> telemetryProperties = new Dictionary <string, string> {
{ "Type", "Azure/Servicebus" }
};
int lockIssues = 0;
#if NET452
Action <BrokeredMessage, IMessage> leaveDeadlLetterInQueue = (deadLetterBrokeredMessage, deadLetterMessage) =>
#endif
#if NETSTANDARD2_0
Action <IMessageReceiver, BrokeredMessage, IMessage> leaveDeadlLetterInQueue = (client, deadLetterBrokeredMessage, deadLetterMessage) =>
#endif
{
// Remove message from queue
try
{
#if NET452
deadLetterBrokeredMessage.Abandon();
#endif
#if NETSTANDARD2_0
client.AbandonAsync(deadLetterBrokeredMessage.SystemProperties.LockToken).Wait(1500);
#endif
lockIssues = 0;
}
catch (MessageLockLostException)
{
lockIssues++;
Logger.LogWarning(string.Format("The lock supplied for abandon for the skipped dead-letter message '{0}' is invalid.", deadLetterBrokeredMessage.MessageId));
}
Logger.LogDebug(string.Format("A dead-letter message of type {0} arrived with the id '{1}' but left in the queue due to settings.", deadLetterMessage.GetType().FullName, deadLetterBrokeredMessage.MessageId));
};
#if NET452
Action <BrokeredMessage> removeDeadlLetterFromQueue = (deadLetterBrokeredMessage) =>
#endif
#if NETSTANDARD2_0
Action <IMessageReceiver, BrokeredMessage> removeDeadlLetterFromQueue = (client, deadLetterBrokeredMessage) =>
#endif
{
// Remove message from queue
try
{
#if NET452
deadLetterBrokeredMessage.Complete();
#endif
#if NETSTANDARD2_0
client.CompleteAsync(deadLetterBrokeredMessage.SystemProperties.LockToken).Wait(1500);
#endif
lockIssues = 0;
}
catch (MessageLockLostException)
{
lockIssues++;
Logger.LogWarning(string.Format("The lock supplied for complete for the skipped dead-letter message '{0}' is invalid.", deadLetterBrokeredMessage.MessageId));
}
Logger.LogDebug(string.Format("A dead-letter message arrived with the id '{0}' but was removed as processing was skipped due to settings.", deadLetterBrokeredMessage.MessageId));
};
Task.Factory.StartNewSafely(() =>
{
int loop = 0;
while (!brokeredMessageRenewCancellationTokenSource.Token.IsCancellationRequested)
{
lockIssues = 0;
IEnumerable <BrokeredMessage> brokeredMessages;
#if NET452
MessagingFactory factory = MessagingFactory.CreateFromConnectionString(ConnectionString);
string deadLetterPath = SubscriptionClient.FormatDeadLetterPath(topicName, topicSubscriptionName);
MessageReceiver client = factory.CreateMessageReceiver(deadLetterPath, ReceiveMode.PeekLock);
brokeredMessages = client.ReceiveBatch(1000);
#endif
#if NETSTANDARD2_0
string deadLetterPath = EntityNameHelper.FormatDeadLetterPath(EntityNameHelper.FormatSubscriptionPath(topicName, topicSubscriptionName));
MessageReceiver client = new MessageReceiver(ConnectionString, deadLetterPath, ReceiveMode.PeekLock);
Task <IList <BrokeredMessage> > receiveTask = client.ReceiveAsync(1000);
receiveTask.Wait(10000);
if (receiveTask.IsCompleted && receiveTask.Result != null)
{
brokeredMessages = receiveTask.Result;
}
else
{
brokeredMessages = Enumerable.Empty <BrokeredMessage>();
}
#endif
foreach (BrokeredMessage brokeredMessage in brokeredMessages)
{
if (lockIssues > 10)
{
break;
}
try
{
Logger.LogDebug(string.Format("A dead-letter message arrived with the id '{0}'.", brokeredMessage.MessageId));
string messageBody = brokeredMessage.GetBodyAsString();
// Closure protection
BrokeredMessage message = brokeredMessage;
try
{
AzureBusHelper.ReceiveEvent
(
messageBody,
@event =>
{
bool isRequired = BusHelper.IsEventRequired(@event.GetType());
if (!isRequired)
{
#if NET452
removeDeadlLetterFromQueue(message);
#endif
#if NETSTANDARD2_0
removeDeadlLetterFromQueue(client, message);
#endif
}
else
{
#if NET452
leaveDeadlLetterInQueue(message, @event);
#endif
#if NETSTANDARD2_0
leaveDeadlLetterInQueue(client, message, @event);
#endif
}
return(true);
},
string.Format("id '{0}'", brokeredMessage.MessageId),
ExtractSignature(message),
SigningTokenConfigurationKey,
() =>
{
#if NET452
removeDeadlLetterFromQueue(message);
#endif
#if NETSTANDARD2_0
removeDeadlLetterFromQueue(client, message);
#endif
},
() => { }
);
}
catch
{
AzureBusHelper.ReceiveCommand
(
messageBody,
command =>
{
bool isRequired = BusHelper.IsEventRequired(command.GetType());
if (!isRequired)
{
#if NET452
removeDeadlLetterFromQueue(message);
#endif
#if NETSTANDARD2_0
removeDeadlLetterFromQueue(client, message);
#endif
}
else
{
#if NET452
leaveDeadlLetterInQueue(message, command);
#endif
#if NETSTANDARD2_0
leaveDeadlLetterInQueue(client, message, command);
#endif
}
return(true);
},
string.Format("id '{0}'", brokeredMessage.MessageId),
ExtractSignature(message),
SigningTokenConfigurationKey,
() =>
{
#if NET452
removeDeadlLetterFromQueue(message);
#endif
#if NETSTANDARD2_0
removeDeadlLetterFromQueue(client, message);
#endif
},
() => { }
);
}
}
catch (Exception exception)
{
TelemetryHelper.TrackException(exception, null, telemetryProperties);
// Indicates a problem, unlock message in queue
Logger.LogError(string.Format("A dead-letter message arrived with the id '{0}' but failed to be process.", brokeredMessage.MessageId), exception: exception);
try
{
#if NET452
brokeredMessage.Abandon();
#endif
#if NETSTANDARD2_0
client.AbandonAsync(brokeredMessage.SystemProperties.LockToken).Wait(1500);
#endif
}
catch (MessageLockLostException)
{
lockIssues++;
Logger.LogWarning(string.Format("The lock supplied for abandon for the skipped dead-letter message '{0}' is invalid.", brokeredMessage.MessageId));
}
}
}
#if NET452
client.Close();
#endif
#if NETSTANDARD2_0
client.CloseAsync().Wait(1500);
#endif
if (loop++ % 5 == 0)
{
loop = 0;
Thread.Yield();
}
else
{
Thread.Sleep(500);
}
}
try
{
brokeredMessageRenewCancellationTokenSource.Dispose();
}
catch (ObjectDisposedException) { }
}, brokeredMessageRenewCancellationTokenSource.Token);
return(brokeredMessageRenewCancellationTokenSource);
}
/// <summary>
/// Receives <see cref="EventData"/> from the command bus.
/// </summary>
protected virtual void ReceiveCommand(PartitionContext context, EventData eventData)
{
DateTimeOffset startedAt = DateTimeOffset.UtcNow;
Stopwatch mainStopWatch = Stopwatch.StartNew();
string responseCode = "200";
// Null means it was skipped
bool?wasSuccessfull = true;
#if NET452
string telemetryName = string.Format("Cqrs/Handle/Command/{0}", eventData.SequenceNumber);
#endif
#if NETSTANDARD2_0
string telemetryName = string.Format("Cqrs/Handle/Command/{0}", eventData.SystemProperties.SequenceNumber);
#endif
ISingleSignOnToken authenticationToken = null;
Guid? guidAuthenticationToken = null;
string stringAuthenticationToken = null;
int? intAuthenticationToken = null;
IDictionary <string, string> telemetryProperties = new Dictionary <string, string> {
{ "Type", "Azure/EventHub" }
};
object value;
if (eventData.Properties.TryGetValue("Type", out value))
{
telemetryProperties.Add("MessageType", value.ToString());
}
TelemetryHelper.TrackMetric("Cqrs/Handle/Command", CurrentHandles++, telemetryProperties);
// Do a manual 10 try attempt with back-off
for (int i = 0; i < 10; i++)
{
try
{
#if NET452
Logger.LogDebug(string.Format("A command message arrived with the partition key '{0}', sequence number '{1}' and offset '{2}'.", eventData.PartitionKey, eventData.SequenceNumber, eventData.Offset));
#endif
#if NETSTANDARD2_0
Logger.LogDebug(string.Format("A command message arrived with the partition key '{0}', sequence number '{1}' and offset '{2}'.", eventData.SystemProperties.PartitionKey, eventData.SystemProperties.SequenceNumber, eventData.SystemProperties.Offset));
#endif
#if NET452
string messageBody = Encoding.UTF8.GetString(eventData.GetBytes());
#endif
#if NETSTANDARD2_0
string messageBody = Encoding.UTF8.GetString(eventData.Body.Array, eventData.Body.Offset, eventData.Body.Count);
#endif
ICommand <TAuthenticationToken> command = AzureBusHelper.ReceiveCommand(null, messageBody, ReceiveCommand,
#if NET452
string.Format("partition key '{0}', sequence number '{1}' and offset '{2}'", eventData.PartitionKey, eventData.SequenceNumber, eventData.Offset),
#endif
#if NETSTANDARD2_0
string.Format("partition key '{0}', sequence number '{1}' and offset '{2}'", eventData.SystemProperties.PartitionKey, eventData.SystemProperties.SequenceNumber, eventData.SystemProperties.Offset),
#endif
ExtractSignature(eventData),
SigningTokenConfigurationKey,
() =>
{
wasSuccessfull = null;
#if NET452
telemetryName = string.Format("Cqrs/Handle/Command/Skipped/{0}", eventData.SequenceNumber);
#endif
#if NETSTANDARD2_0
telemetryName = string.Format("Cqrs/Handle/Command/Skipped/{0}", eventData.SystemProperties.SequenceNumber);
#endif
responseCode = "204";
// Remove message from queue
context.CheckpointAsync(eventData);
#if NET452
Logger.LogDebug(string.Format("A command message arrived with the partition key '{0}', sequence number '{1}' and offset '{2}' but processing was skipped due to command settings.", eventData.PartitionKey, eventData.SequenceNumber, eventData.Offset));
#endif
#if NETSTANDARD2_0
Logger.LogDebug(string.Format("A command message arrived with the partition key '{0}', sequence number '{1}' and offset '{2}' but processing was skipped due to command settings.", eventData.SystemProperties.PartitionKey, eventData.SystemProperties.SequenceNumber, eventData.SystemProperties.Offset));
#endif
TelemetryHelper.TrackEvent("Cqrs/Handle/Command/Skipped", telemetryProperties);
}
);
if (wasSuccessfull != null)
{
if (command != null)
{
telemetryName = string.Format("{0}/{1}", command.GetType().FullName, command.Id);
authenticationToken = command.AuthenticationToken as ISingleSignOnToken;
if (AuthenticationTokenIsGuid)
{
guidAuthenticationToken = command.AuthenticationToken as Guid?;
}
if (AuthenticationTokenIsString)
{
stringAuthenticationToken = command.AuthenticationToken as string;
}
if (AuthenticationTokenIsInt)
{
intAuthenticationToken = command.AuthenticationToken as int?;
}
var telemeteredMessage = command as ITelemeteredMessage;
if (telemeteredMessage != null)
{
telemetryName = telemeteredMessage.TelemetryName;
}
telemetryName = string.Format("Cqrs/Handle/Command/{0}", telemetryName);
}
// Remove message from queue
context.CheckpointAsync(eventData);
}
#if NET452
Logger.LogDebug(string.Format("A command message arrived and was processed with the partition key '{0}', sequence number '{1}' and offset '{2}'.", eventData.PartitionKey, eventData.SequenceNumber, eventData.Offset));
#endif
#if NETSTANDARD2_0
Logger.LogDebug(string.Format("A command message arrived and was processed with the partition key '{0}', sequence number '{1}' and offset '{2}'.", eventData.SystemProperties.PartitionKey, eventData.SystemProperties.SequenceNumber, eventData.SystemProperties.Offset));
#endif
wasSuccessfull = true;
responseCode = "200";
return;
}
catch (UnAuthorisedMessageReceivedException exception)
{
TelemetryHelper.TrackException(exception, null, telemetryProperties);
// Indicates a problem, unlock message in queue
#if NET452
Logger.LogError(string.Format("A command message arrived with the partition key '{0}', sequence number '{1}' and offset '{2}' but was not authorised.", eventData.PartitionKey, eventData.SequenceNumber, eventData.Offset), exception: exception);
#endif
#if NETSTANDARD2_0
Logger.LogError(string.Format("A command message arrived with the partition key '{0}', sequence number '{1}' and offset '{2}' but was not authorised.", eventData.SystemProperties.PartitionKey, eventData.SystemProperties.SequenceNumber, eventData.SystemProperties.Offset), exception: exception);
#endif
wasSuccessfull = false;
responseCode = "401";
telemetryProperties.Add("ExceptionType", exception.GetType().FullName);
telemetryProperties.Add("ExceptionMessage", exception.Message);
}
catch (NoHandlersRegisteredException exception)
{
TelemetryHelper.TrackException(exception, null, telemetryProperties);
// Indicates a problem, unlock message in queue
#if NET452
Logger.LogError(string.Format("A command message arrived with the partition key '{0}', sequence number '{1}' and offset '{2}' but no handlers were found to process it.", eventData.PartitionKey, eventData.SequenceNumber, eventData.Offset), exception: exception);
#endif
#if NETSTANDARD2_0
Logger.LogError(string.Format("A command message arrived with the partition key '{0}', sequence number '{1}' and offset '{2}' but no handlers were found to process it.", eventData.SystemProperties.PartitionKey, eventData.SystemProperties.SequenceNumber, eventData.SystemProperties.Offset), exception: exception);
#endif
wasSuccessfull = false;
responseCode = "501";
telemetryProperties.Add("ExceptionType", exception.GetType().FullName);
telemetryProperties.Add("ExceptionMessage", exception.Message);
}
catch (NoHandlerRegisteredException exception)
{
TelemetryHelper.TrackException(exception, null, telemetryProperties);
// Indicates a problem, unlock message in queue
#if NET452
Logger.LogError(string.Format("A command message arrived with the partition key '{0}', sequence number '{1}' and offset '{2}' but no handler was found to process it.", eventData.PartitionKey, eventData.SequenceNumber, eventData.Offset), exception: exception);
#endif
#if NETSTANDARD2_0
Logger.LogError(string.Format("A command message arrived with the partition key '{0}', sequence number '{1}' and offset '{2}' but no handler was found to process it.", eventData.SystemProperties.PartitionKey, eventData.SystemProperties.SequenceNumber, eventData.SystemProperties.Offset), exception: exception);
#endif
wasSuccessfull = false;
responseCode = "501";
telemetryProperties.Add("ExceptionType", exception.GetType().FullName);
telemetryProperties.Add("ExceptionMessage", exception.Message);
}
catch (Exception exception)
{
// Indicates a problem, unlock message in queue
#if NET452
Logger.LogError(string.Format("A command message arrived with the partition key '{0}', sequence number '{1}' and offset '{2}' but failed to be process.", eventData.PartitionKey, eventData.SequenceNumber, eventData.Offset), exception: exception);
#endif
#if NETSTANDARD2_0
Logger.LogError(string.Format("A command message arrived with the partition key '{0}', sequence number '{1}' and offset '{2}' but failed to be process.", eventData.SystemProperties.PartitionKey, eventData.SystemProperties.SequenceNumber, eventData.SystemProperties.Offset), exception: exception);
#endif
switch (i)
{
case 0:
case 1:
// 10 seconds
Thread.Sleep(10 * 1000);
break;
case 2:
case 3:
// 30 seconds
Thread.Sleep(30 * 1000);
break;
case 4:
case 5:
case 6:
// 1 minute
Thread.Sleep(60 * 1000);
break;
case 7:
case 8:
// 3 minutes
Thread.Sleep(3 * 60 * 1000);
break;
case 9:
telemetryProperties.Add("ExceptionType", exception.GetType().FullName);
telemetryProperties.Add("ExceptionMessage", exception.Message);
break;
}
wasSuccessfull = false;
responseCode = "500";
}
finally
{
// Eventually just accept it
context.CheckpointAsync(eventData);
TelemetryHelper.TrackMetric("Cqrs/Handle/Command", CurrentHandles--, telemetryProperties);
mainStopWatch.Stop();
if (guidAuthenticationToken != null)
{
TelemetryHelper.TrackRequest
(
telemetryName,
guidAuthenticationToken,
startedAt,
mainStopWatch.Elapsed,
responseCode,
wasSuccessfull == null || wasSuccessfull.Value,
telemetryProperties
);
}
else if (intAuthenticationToken != null)
{
TelemetryHelper.TrackRequest
(
telemetryName,
intAuthenticationToken,
startedAt,
mainStopWatch.Elapsed,
responseCode,
wasSuccessfull == null || wasSuccessfull.Value,
telemetryProperties
);
}
else if (stringAuthenticationToken != null)
{
TelemetryHelper.TrackRequest
(
telemetryName,
stringAuthenticationToken,
startedAt,
mainStopWatch.Elapsed,
responseCode,
wasSuccessfull == null || wasSuccessfull.Value,
telemetryProperties
);
}
else
{
TelemetryHelper.TrackRequest
(
telemetryName,
authenticationToken,
startedAt,
mainStopWatch.Elapsed,
responseCode,
wasSuccessfull == null || wasSuccessfull.Value,
telemetryProperties
);
}
TelemetryHelper.Flush();
}
}
}
