private void TrackMessageLags(IBrokeredMessage brokeredMessage)
{
_telemetryService.TrackMessageDeliveryLag <TMessage>(DateTimeOffset.UtcNow - brokeredMessage.ScheduledEnqueueTimeUtc);
// we expect the "enqueue lag" to be zero or really close to zero pretty much all the time, logging it just in case it is not
// and for historical perspective if we need one.
_telemetryService.TrackEnqueueLag <TMessage>(brokeredMessage.EnqueuedTimeUtc - brokeredMessage.ScheduledEnqueueTimeUtc);
}
private IMessageHandler CreateMessageHandler(IBrokeredMessage message)
{
var messageType = message.Properties[MessageProperties.MessageType].ToString();
var messageHandlerType = _messageHandlerRegistry.GetMessageTypeHandler(messageType);
return((IMessageHandler)_typeCreator.Create(messageHandlerType));
}
private void ConsumeMessage(SubscriberInfo subscriberInfo, IBrokeredMessage brokeredMessage)
{
try
{
var messageBody = brokeredMessage.GetBody <string>();
var message = JsonConvert.DeserializeObject <TMessage>(messageBody);
_logService.Debug("Consuming message id '{0}', topic/subscription '{1}', body '{2}'",
brokeredMessage.MessageId, subscriberInfo.SubscriptionPath, messageBody);
var state = subscriberInfo.Subscriber.Consume(message);
HandleMessageResult(subscriberInfo, state, brokeredMessage, messageBody);
_logService.Debug("Consumed message id '{0}', topic/subscription '{1}', body '{2}'",
brokeredMessage.MessageId, subscriberInfo.SubscriptionPath, messageBody);
}
catch (Exception e)
{
var deadLetterReason = string.Format(
"Unexpected exception consuming message id '{0}, topic/subscription '{1}': message will be dead-lettered",
brokeredMessage.MessageId, subscriberInfo.SubscriptionPath);
_logService.Error(deadLetterReason, e);
brokeredMessage.DeadLetter();
}
}
public PackageValidationMessageData DeserializePackageValidationMessageData(IBrokeredMessage message)
{
var schemaName = message.GetSchemaName();
switch (schemaName)
{
case ProcessValidationSetSchemaName:
var processValidationSet = _processValidationSetSerializer.Deserialize(message);
return(new PackageValidationMessageData(
PackageValidationMessageType.ProcessValidationSet,
processValidationSet: new ProcessValidationSetData(
processValidationSet.PackageId,
processValidationSet.PackageVersion,
processValidationSet.ValidationTrackingId,
processValidationSet.ValidatingType,
processValidationSet.EntityKey),
checkValidator: null,
deliveryCount: message.DeliveryCount));
case CheckValidatorSchemaName:
var checkValidator = _checkValidatorSerializer.Deserialize(message);
return(new PackageValidationMessageData(
PackageValidationMessageType.CheckValidator,
processValidationSet: null,
checkValidator: new CheckValidatorData(checkValidator.ValidationId),
deliveryCount: message.DeliveryCount));
default:
throw new FormatException($"The provided schema name '{schemaName}' is not supported.");
}
}
public ScanAndSignEnqueuerFactsBase()
{
_topicClientMock = new Mock <ITopicClient>();
_serializerMock = new Mock <IBrokeredMessageSerializer <ScanAndSignMessage> >();
_logger = Mock.Of <ILogger <ScanAndSignEnqueuer> >();
_configurationAccessorMock = new Mock <IOptionsSnapshot <ScanAndSignEnqueuerConfiguration> >();
_configuration = new ScanAndSignEnqueuerConfiguration();
_configurationAccessorMock
.SetupGet(ca => ca.Value)
.Returns(_configuration);
_target = new ScanAndSignEnqueuer(
_topicClientMock.Object,
_serializerMock.Object,
_configurationAccessorMock.Object,
_logger);
_validationRequest = new NuGetValidationRequest(Guid.NewGuid(), 42, "somepackage", "someversion", "https://example.com/testpackage.nupkg");
_owners = new List <string> {
"Billy", "Bob"
};
_serializedMessage = new BrokeredMessageWrapper("somedata");
_serializerMock
.Setup(s => s.Serialize(It.IsAny <ScanAndSignMessage>()))
.Callback <ScanAndSignMessage>(m => _capturedMessage = m)
.Returns(_serializedMessage);
_topicClientMock
.Setup(tc => tc.SendAsync(It.IsAny <IBrokeredMessage>()))
.Callback <IBrokeredMessage>(m => _capturedBrokeredMessage = m)
.Returns(Task.CompletedTask);
}
public void OnMessageHandler(IBrokeredMessage msg)
{
//receive loop begin
bool failed = false;
if (msg != null)
{
// Make sure we are not told to stop receiving while we were waiting for a new message.
if (!data.CancelToken.IsCancellationRequested)
{
// Process the received message.
logger.Debug("ProcessMessagesForSubscription Start received new message: {0}", data.EndPointData.SubscriptionNameDebug);
var receiveState = new AzureReceiveState(data, methodInfo, serializer, msg);
failed = OnReceiveProcess(receiveState);
logger.Debug("ProcessMessagesForSubscription End received new message: {0}", data.EndPointData.SubscriptionNameDebug);
}
}
if (data.CancelToken.IsCancellationRequested)
{
//Cancel the message pump.
data.SetMessageLoopCompleted();
try {
if (!data.Client.IsClosed)
{
data.Client.Close();
}
}
catch {
}
}
else if (failed)
{
if (data.EndPointData.AttributeData.PauseTimeIfErrorWasThrown > 0)
{
//For now, do not support pause time
Thread.Sleep(1000); //This has zero impact if the thread count is > 1
//lock (lockObject) {
// if (DateTime.Now.AddMilliseconds(-data.EndPointData.AttributeData.PauseTimeIfErrorWasThrown) >= lastResetTime) {
// retryPolicy.ExecuteAction(() => {
// if (!data.Client.IsClosed) {
// data.Client.Close();
// }
// });
// Thread.Sleep(data.EndPointData.AttributeData.PauseTimeIfErrorWasThrown);
// lastResetTime = DateTime.Now;
// retryPolicy.ExecuteAction(() => {
// Configure(endpoint);
// data.Client.OnMessage(OnMessageHandler, options);
// });
// }
//}
}
else
{
Thread.Sleep(1000); //This has zero impact if the thread count is > 1
}
}
}
public void Send(IBrokeredMessage message)
{
_callback.Invoke(message);
if (_onMessageOptions.AutoComplete)
{
message.Complete();
}
}
public AccountDeleteMessage Deserialize(IBrokeredMessage brokeredMessage)
{
var message = _serializer.Deserialize(brokeredMessage);
return(new AccountDeleteMessage(
message.Username,
message.Source));
}
public IAsyncResult BeginSend(IBrokeredMessage message, AsyncCallback callback, object state) {
var retVal = new MockIAsyncResult() {
AsyncState = state
};
_messages[retVal] = message;
callback(retVal);
return retVal;
}
public IAsyncResult BeginSend(IBrokeredMessage message, AsyncCallback callback, object state)
{
if (!(message is BrokeredMessageWrapper))
{
throw new ArgumentOutOfRangeException("message", "message must be BrokeredMessage for Azure use");
}
return(topicClient.BeginSend((message as BrokeredMessageWrapper).GetMessage(), callback, state));
}
public CertificateValidationMessage Deserialize(IBrokeredMessage brokeredMessage)
{
var message = _serializer.Deserialize(brokeredMessage);
return(new CertificateValidationMessage(
message.CertificateKey,
message.ValidationId,
message.RevalidateRevokedCertificate));
}
public ReceivedMessage(IBrokeredMessage brokeredMessage, T message, IDictionary <string, object> metadata)
{
Guard.ArgumentNotNull(brokeredMessage, "brokeredMessage");
Guard.ArgumentNotNull(message, "message");
Guard.ArgumentNotNull(metadata, "metadata");
this.brokeredMessage = brokeredMessage;
this.message = message;
this.metadata = metadata;
}
public void Send(IBrokeredMessage message)
{
++SendAsyncCallCount;
if (ShouldFail)
{
throw new Exception();
}
LastSentMessage = message;
}
public SignatureValidationMessage Deserialize(IBrokeredMessage message)
{
var deserializedMessage = _serializer.Deserialize(message);
return(new SignatureValidationMessage(
deserializedMessage.PackageId,
deserializedMessage.PackageVersion,
deserializedMessage.NupkgUri,
deserializedMessage.ValidationId));
}
public void EndSend(IAsyncResult result)
{
IBrokeredMessage message = null;
if (!_messages.TryGetValue(result, out message))
{
throw new ApplicationException("You must call EndSend with a valid IAsyncResult. Duplicate Calls are not allowed.");
}
serviceBus.SendMessage(message);
}
public void MessageDeadLetter(IBrokeredMessage message, string deadLetterReason, string deadLetterErrorDescription)
{
foreach (var subscription in _subscriptions)
{
if (subscription.Messages.Contains(message))
{
subscription.Messages.Remove(message);
break;
}
}
}
public void MessageComplete(IBrokeredMessage message)
{
foreach (var subscription in _subscriptions)
{
if (subscription.Messages.Contains(message))
{
subscription.Messages.Remove(message);
break;
}
}
}
public IAsyncResult BeginSend(IBrokeredMessage message, AsyncCallback callback, object state)
{
var retVal = new MockIAsyncResult()
{
AsyncState = state
};
_messages[retVal] = message;
callback(retVal);
return(retVal);
}
public ScanAndSignMessage Deserialize(IBrokeredMessage message)
{
var deserializedMessage = _serializer.Deserialize(message);
return(new ScanAndSignMessage(
deserializedMessage.OperationRequestType,
deserializedMessage.PackageValidationId,
deserializedMessage.BlobUri,
deserializedMessage.V3ServiceIndexUrl,
deserializedMessage.Owners));
}
public SymbolsValidatorMessage Deserialize(IBrokeredMessage message)
{
var deserializedMessage = _serializer.Deserialize(message);
return(new SymbolsValidatorMessage(
deserializedMessage.ValidationId,
deserializedMessage.SymbolsPackageKey,
deserializedMessage.PackageId,
deserializedMessage.PackageNormalizedVersion,
deserializedMessage.SnupkgUrl));
}
public Task SendAsync(IBrokeredMessage message)
{
++SendAsyncCallCount;
if (ShouldFail)
{
throw new Exception();
}
LastSentMessage = message;
return(Task.FromResult(0));
}
public PackageValidationMessageData DeserializePackageValidationMessageData(IBrokeredMessage message)
{
var data = _serializer.Deserialize(message);
return(new PackageValidationMessageData(
data.PackageId,
data.PackageVersion,
data.ValidationTrackingId,
data.ValidatingType,
message.DeliveryCount));
}
public AzureReceiveState(AzureBusReceiverState data, MethodInfo methodInfo,
IServiceBusSerializer serializer, IBrokeredMessage message) {
Guard.ArgumentNotNull(data, "data");
Guard.ArgumentNotNull(methodInfo, "methodInfo");
Guard.ArgumentNotNull(serializer, "serializer");
Guard.ArgumentNotNull(message, "message");
this.Data = data;
this.MethodInfo = methodInfo;
this.Serializer = serializer;
this.Message = message;
}
public AzureReceiveState(AzureBusReceiverState data, MethodInfo methodInfo,
IServiceBusSerializer serializer, IBrokeredMessage message)
{
Guard.ArgumentNotNull(data, "data");
Guard.ArgumentNotNull(methodInfo, "methodInfo");
Guard.ArgumentNotNull(serializer, "serializer");
Guard.ArgumentNotNull(message, "message");
this.Data = data;
this.MethodInfo = methodInfo;
this.Serializer = serializer;
this.Message = message;
}
private static void AssertTypeAndSchemaVersion(IBrokeredMessage message, string type, int schemaVersion)
{
if (GetType(message) != type)
{
throw new FormatException($"The provided message should have {SchemaNameKey} property '{type}'.");
}
if (GetSchemaVersion(message) != schemaVersion)
{
throw new FormatException($"The provided message should have {SchemaVersionKey} property '{schemaVersion}'.");
}
}
public async Task HandleAsync(IBrokeredMessage message)
{
try
{
var handler = CreateMessageHandler(message);
await handler.HandleAsync(message);
}
catch (Exception exception)
{
//need to log here
throw;
}
}
private static T GetProperty <T>(IBrokeredMessage message, string key, string typeLabel)
{
object value;
if (!message.Properties.TryGetValue(key, out value))
{
throw new FormatException($"The provided message does not have a {key} property.");
}
if (!(value is T))
{
throw new FormatException($"The provided message contains a {key} property that is not {typeLabel}.");
}
return((T)value);
}
public EmailMessageData DeserializeEmailMessageData(IBrokeredMessage message)
{
var data = _serializer.Deserialize(message);
return(new EmailMessageData(
data.Subject,
data.PlainTextBody,
data.HtmlBody,
data.Sender,
data.To,
data.CC,
data.Bcc,
data.ReplyTo,
data.MessageTrackingId,
message.DeliveryCount));
}
private async Task OnMessageAsync(IBrokeredMessage brokeredMessage)
{
if (!_running)
{
_logger.LogWarning("Dropping message from Service Bus as shutdown has been initiated");
return;
}
Interlocked.Increment(ref _numberOfMessagesInProgress);
TrackMessageLags(brokeredMessage);
using (var scope = _logger.BeginScope($"{nameof(SubscriptionProcessor<TMessage>)}.{nameof(OnMessageAsync)} {{CallGuid}} {{CallStartTimestamp}}",
Guid.NewGuid(),
DateTimeOffset.UtcNow.ToString("O")))
{
try
{
_logger.LogInformation("Received message from Service Bus subscription, processing");
var message = _serializer.Deserialize(brokeredMessage);
if (await _handler.HandleAsync(message))
{
_logger.LogInformation("Message was successfully handled, marking the brokered message as completed");
await brokeredMessage.CompleteAsync();
}
else
{
_logger.LogInformation("Handler did not finish processing message, requeueing message to be reprocessed");
}
}
catch (Exception e)
{
_logger.LogError(Event.SubscriptionMessageHandlerException, e, "Requeueing message as it was unsuccessfully processed due to exception");
// exception should not be propagated to the topic client, because it will
// abandon the message and will cause the retry to happen immediately, which,
// in turn, have higher chances of failing again if we, for example, experiencing
// transitive network issues.
}
finally
{
Interlocked.Decrement(ref _numberOfMessagesInProgress);
}
}
}
public Task SendAsync(IBrokeredMessage message)
{
// For now, assume the only implementation is the wrapper type. We could clone over all properties
// that the interface supports, but this is not necessary right now.
var wrapper = message as BrokeredMessageWrapper;
BrokeredMessage innerMessage;
if (message != null)
{
innerMessage = wrapper.BrokeredMessage;
}
else
{
throw new ArgumentException(
$"The message must be of type {typeof(BrokeredMessageWrapper).FullName}.",
nameof(message));
}
return(_client.SendAsync(innerMessage));
}
static bool SafeComplete(IBrokeredMessage msg)
{
try {
// Mark brokered message as complete.
msg.Complete();
// Return a result indicating that the message has been completed successfully.
return(true);
}
catch (MessageLockLostException) {
// It's too late to compensate the loss of a message lock. We should just ignore it so that it does not break the receive loop.
// We should be prepared to receive the same message again.
}
catch (MessagingException) {
// There is nothing we can do as the connection may have been lost, or the underlying topic/subscription may have been removed.
// If Complete() fails with this exception, the only recourse is to prepare to receive another message (possibly the same one).
}
return(false);
}
/// <summary>
/// Send a message onto the service bus to any item that is registered for the type
/// </summary>
/// <param name="message"></param>
public void SendMessage(IBrokeredMessage message)
{
var typeName = message.Properties[AzureSenderReceiverBase.TYPE_HEADER_NAME] as string;
if (typeName == null)
{
throw new ApplicationException(AzureSenderReceiverBase.TYPE_HEADER_NAME + " Key does not exist.");
}
typeName = typeName.Replace("_", ".");
//NOTE Limit is test class must exist in this assembly for now.
var theType = this.GetType().Assembly.GetType(typeName);
var handlers = _subscriptions.Where(item => item.Type == theType).ToList();
foreach (var handler in handlers)
{
handler.AddMessage(new MockBrokeredMessage(this, message));
}
}
public IAsyncResult BeginSend(IBrokeredMessage message, AsyncCallback callback, object state) {
if (!(message is BrokeredMessageWrapper)) {
throw new ArgumentOutOfRangeException("message", "message must be BrokeredMessage for Azure use");
}
return topicClient.BeginSend((message as BrokeredMessageWrapper).GetMessage(), callback, state);
}
public void OnMessageHandler(IBrokeredMessage msg) {
//receive loop begin
bool failed = false;
if (msg != null) {
// Make sure we are not told to stop receiving while we were waiting for a new message.
if (!data.CancelToken.IsCancellationRequested) {
// Process the received message.
logger.Debug("ProcessMessagesForSubscription Start received new message: {0}", data.EndPointData.SubscriptionNameDebug);
var receiveState = new AzureReceiveState(data, methodInfo, serializer, msg);
failed = OnReceiveProcess(receiveState);
logger.Debug("ProcessMessagesForSubscription End received new message: {0}", data.EndPointData.SubscriptionNameDebug);
}
}
if (data.CancelToken.IsCancellationRequested) {
//Cancel the message pump.
data.SetMessageLoopCompleted();
try {
if (!data.Client.IsClosed) {
data.Client.Close();
}
}
catch {
}
}
else if (failed) {
if (data.EndPointData.AttributeData.PauseTimeIfErrorWasThrown > 0) {
//For now, do not support pause time
Thread.Sleep(1000); //This has zero impact if the thread count is > 1
//lock (lockObject) {
// if (DateTime.Now.AddMilliseconds(-data.EndPointData.AttributeData.PauseTimeIfErrorWasThrown) >= lastResetTime) {
// retryPolicy.ExecuteAction(() => {
// if (!data.Client.IsClosed) {
// data.Client.Close();
// }
// });
// Thread.Sleep(data.EndPointData.AttributeData.PauseTimeIfErrorWasThrown);
// lastResetTime = DateTime.Now;
// retryPolicy.ExecuteAction(() => {
// Configure(endpoint);
// data.Client.OnMessage(OnMessageHandler, options);
// });
// }
//}
}
else {
Thread.Sleep(1000); //This has zero impact if the thread count is > 1
}
}
}
static bool SafeComplete(IBrokeredMessage msg) {
try {
// Mark brokered message as complete.
msg.Complete();
// Return a result indicating that the message has been completed successfully.
return true;
}
catch (MessageLockLostException) {
// It's too late to compensate the loss of a message lock. We should just ignore it so that it does not break the receive loop.
// We should be prepared to receive the same message again.
}
catch (MessagingException) {
// There is nothing we can do as the connection may have been lost, or the underlying topic/subscription may have been removed.
// If Complete() fails with this exception, the only recourse is to prepare to receive another message (possibly the same one).
}
return false;
}
static bool SafeAbandon(IBrokeredMessage msg) {
try {
// Abandons a brokered message. This will cause the Service Bus to unlock the message and make it available to be received again,
// either by the same consumer or by another competing consumer.
msg.Abandon();
// Return a result indicating that the message has been abandoned successfully.
return true;
}
catch (MessageLockLostException) {
// It's too late to compensate the loss of a message lock. We should just ignore it so that it does not break the receive loop.
// We should be prepared to receive the same message again.
}
catch (MessagingException) {
// There is nothing we can do as the connection may have been lost, or the underlying topic/subscription may have been removed.
// If Abandon() fails with this exception, the only recourse is to receive another message (possibly the same one).
}
return false;
}
public void OnMessageHandler(IBrokeredMessage msg) {
//receive loop begin
bool failed = false;
if (msg != null) {
// Make sure we are not told to stop receiving while we were waiting for a new message.
if (!data.CancelToken.IsCancellationRequested) {
// Process the received message.
logger.Debug("ProcessMessagesForSubscription Start received new message: {0}", data.EndPointData.SubscriptionNameDebug);
var receiveState = new AzureReceiveState(data, methodInfo, serializer, msg);
failed = OnReceiveProcess(receiveState);
logger.Debug("ProcessMessagesForSubscription End received new message: {0}", data.EndPointData.SubscriptionNameDebug);
}
}
if (data.CancelToken.IsCancellationRequested) {
//Cancel the message pump.
data.SetMessageLoopCompleted();
try {
data.Client.Close();
}
catch {
}
}
else if (failed) {
//close the pump.
data.Client.Close();
if (data.EndPointData.AttributeData.PauseTimeIfErrorWasThrown > 0) {
Thread.Sleep(data.EndPointData.AttributeData.PauseTimeIfErrorWasThrown);
}
else {
Thread.Sleep(1000);
}
data.Client.OnMessage(OnMessageHandler, options);
}
}
