private IServiceBusCreationOptions GetServiceBusCreationOptions()
{
var queueMapping = AutoMessageMapper.GetMapping <T>();
var creationOptions = queueMapping as IServiceBusCreationOptions;
return(creationOptions ?? _configuration.DefaultCreationOptions);
}
private async Task <ServiceBusMessage> CreateMessageAsync <T>(T body) where T : IMessage
{
var serializer = GetSerializer <T>();
var message = new ServiceBusMessage(serializer.Serialize(body))
{
ContentType = serializer.ContentType
};
if (typeof(ICommandWithAttachment).IsAssignableFrom(typeof(T)))
{
if (_attachmentProvider == null)
{
throw new AttachmentProviderMissingException();
}
var attachmentMessage = (ICommandWithAttachment)body;
if (attachmentMessage.Attachment != null)
{
var attachmentIds = new List <string>();
var id = Guid.NewGuid().ToString("N");
var queueName = AutoMessageMapper.GetQueueName <T>();
await _attachmentProvider.UploadAttachmentAsync(queueName, id, attachmentMessage.Attachment).ConfigureAwait(false);
attachmentIds.Add(id);
message.ApplicationProperties[AttachmentUtility.AttachmentKey] = string.Join(",", attachmentIds);
}
}
return(message);
}
public async Task Should_attach_attachment_when_command_have_attachment()
{
//arrange
var message = new AttachmentCommand();
var nextProcessor = new Mock <IMessageProcessor>();
var stream = new MemoryStream(Encoding.UTF8.GetBytes("this is a stream"));
var attachment = new MessageAttachment("test.txt", "text/plain", stream);
var stateHandler = new Mock <IMessageStateHandler <AttachmentCommand> >();
stateHandler.Setup(x => x.GetMessageAsync()).ReturnsAsync(message);
stateHandler.Setup(x => x.MessageProperties).Returns(new Dictionary <string, string> {
{ AttachmentUtility.AttachmentKey, "89BDF3DB-896C-448D-A84E-872CBA8DBC9F" }
});
var attachmentProvider = new Mock <IMessageAttachmentProvider>();
attachmentProvider.Setup(x => x.GetAttachmentAsync(AutoMessageMapper.GetQueueName <AttachmentCommand>(), It.IsAny <string>(), It.IsAny <CancellationToken>()))
.ReturnsAsync(attachment);
var middleware = new AttachmentMiddleware(attachmentProvider.Object);
//act
await middleware.ProcessAsync(stateHandler.Object, Mock.Of <IPipelineInformation>(), nextProcessor.Object, CancellationToken.None);
//assert
stream.CanRead.Should().BeFalse("It should have been disposed");
message.Attachment.Filename.Should().Be("test.txt");
message.Attachment.ContentType.Should().Be("text/plain");
nextProcessor.Verify(x => x.ProcessAsync(stateHandler.Object, CancellationToken.None), Times.Once);
}
public override async Task StartAsync(CancellationToken cancellationToken)
{
var db = ConnectionMultiplexer.GetDatabase(_redisConfiguration.DatabaseId);
await db.SetAddAsync(RedisQueueConventions.GetSubscriptionKey(AutoMessageMapper.GetQueueName <T>()), _subscription.Name).ConfigureAwait(false);
await base.StartAsync(cancellationToken).ConfigureAwait(false);
}
public async Task ProcessAsync <T>(IMessageStateHandler <T> messageStateHandler, IPipelineInformation pipelineInformation, IMessageProcessor next, CancellationToken cancellationToken) where T : class, IMessage
{
IMessageAttachment attachment = null;
var queueName = AutoMessageMapper.GetQueueName <T>();
try
{
string attachmentId = null;
if (typeof(ICommandWithAttachment).IsAssignableFrom(typeof(T)))
{
attachmentId = AttachmentUtility.GetAttachmentIds(messageStateHandler.MessageProperties).FirstOrDefault();
if (!string.IsNullOrEmpty(attachmentId))
{
attachment = await _attachmentProvider.GetAttachmentAsync(queueName, attachmentId, cancellationToken).ConfigureAwait(false);
var message = (ICommandWithAttachment)await messageStateHandler.GetMessageAsync().ConfigureAwait(false);
message.Attachment = attachment;
}
}
await next.ProcessAsync(messageStateHandler, cancellationToken).ConfigureAwait(false);
if (attachment != null)
{
attachment.Stream?.Dispose();
await _attachmentProvider.DeleteAttachmentAsync(queueName, attachmentId, cancellationToken).ConfigureAwait(false);
}
}
finally
{
attachment?.Stream?.Dispose();
}
}
private ServiceBusProcessor CreateReceiverClient <T>(ServiceBusProcessorOptions options) where T : ICommand
{
var queueName = AutoMessageMapper.GetQueueName <T>();
var client = new ServiceBusClient(_connectionString);
return(client.CreateProcessor(queueName, options));
}
private ServiceBusProcessor CreateSubscriptionClient <T>(string subscriptionName, ServiceBusProcessorOptions options) where T : IEvent
{
var topicName = AutoMessageMapper.GetQueueName <T>();
var client = new ServiceBusClient(_connectionString);
return(client.CreateProcessor(topicName, subscriptionName, options));
}
private ServiceBusSender CreateQueueClient <T>() where T : IMessage
{
var queueName = AutoMessageMapper.GetQueueName <T>();
var client = new ServiceBusClient(_connectionString);
return(client.CreateSender(queueName));
}
public async Task StartAsync(CancellationToken cancellationToken)
{
_cancellationToken = cancellationToken;
_client = await _clientFactory.GetReceiverClient <T>(new ServiceBusProcessorOptions
{
AutoCompleteMessages = false,
MaxAutoLockRenewalDuration = Settings.MessageLockTimeout,
MaxConcurrentCalls = Settings.MaxConcurrentCalls,
PrefetchCount = Settings.PrefetchCount,
ReceiveMode = ServiceBusReceiveMode.PeekLock
}).ConfigureAwait(false);
var queueName = AutoMessageMapper.GetQueueName <T>();
if (!await _managementClient.QueueExistsAsync(queueName, _cancellationToken).ConfigureAwait(false))
{
try
{
var serviceBusCreationOptions = GetServiceBusCreationOptions();
await _managementClient.CreateQueueAsync(new CreateQueueOptions(queueName)
{
EnablePartitioning = serviceBusCreationOptions.EnablePartitioning,
EnableBatchedOperations = serviceBusCreationOptions.EnableBatchedOperations
}, cancellationToken).ConfigureAwait(false);
}
catch (ServiceBusException e)
{
_log.Error(e, "Failed to create queue {QueueName}", queueName);
throw;
}
}
_deadLetterLimit = Settings.DeadLetterDeliveryLimit;
_client.ProcessMessageAsync += ClientOnProcessMessageAsync;
_client.ProcessErrorAsync += ClientOnProcessErrorAsync;
await _client.StartProcessingAsync(cancellationToken);
#pragma warning disable 4014
// ReSharper disable once MethodSupportsCancellation
Task.Run(async() =>
{
_cancellationToken.WaitHandle.WaitOne();
//Cancellation requested
try
{
_log.Information($"Closing ServiceBus channel receiver for {typeof(T).Name}");
await _client.CloseAsync(CancellationToken.None);
}
catch (Exception)
{
//Swallow
}
});
#pragma warning restore 4014
}
private async Task Initialize()
{
var queueName = AutoMessageMapper.GetQueueName <T>();
_storageQueueClient = new StorageQueueClient(_storageOptions, queueName);
await _storageQueueClient.CreateIfNotExistsAsync().ConfigureAwait(false);
_messagePump = new StorageQueueMessagePump(_storageQueueClient, Settings, _hostConfiguration.Log);
}
private IMessageSerializer GetSerializer(ITransportChannelFactory channelFactory, Type messageType)
{
var mapping = AutoMessageMapper.GetMapping(messageType);
if (mapping is ICustomMessageSerializer serializer)
{
return(serializer.MessageSerializer);
}
return(channelFactory.Configuration.MessageSerializer);
}
public async Task PublishAsync <T>(T message) where T : IRedisEvent
{
var db = _multiplexer.GetDatabase(_configuration.DatabaseId);
var queueName = AutoMessageMapper.GetQueueName <T>();
var subscriptions = await db.SetMembersAsync(RedisQueueConventions.GetSubscriptionKey(queueName)).ConfigureAwait(false);
if (subscriptions == null)
{
return;
}
await Task.WhenAll(subscriptions.Select(sub => SendAsync(message, RedisQueueConventions.GetSubscriptionQueueName(queueName, sub)))).ConfigureAwait(false);
}
private IMessageSerializer GetSerializer <T>() where T : IMessage
{
return(_serializers.GetOrAdd(typeof(T), type =>
{
var mapper = AutoMessageMapper.GetMapping <T>();
if (mapper is ICustomMessageSerializer serializer)
{
return serializer.MessageSerializer;
}
return _configuration.MessageSerializer;
}));
}
private IStorageQueueClient GetClient <T>() where T : class, ICommand
{
return(_queueClients.GetOrAdd(typeof(T), type =>
{
var serializer = _options.MessageSerializer;
var mapping = AutoMessageMapper.GetMapping <T>();
if (mapping is ICustomMessageSerializer customSerializer)
{
serializer = customSerializer.MessageSerializer;
}
return new StorageQueueClient(_options, serializer, _attachmentProvider, AutoMessageMapper.GetQueueName <T>());
}));
}
internal async Task PumpAsync <T>(Func <StorageQueueMessage, CancellationToken, Task> action) where T : IStorageQueueCommand
{
var messagesFound = false;
try
{
var queueName = AutoMessageMapper.GetQueueName <T>();
var prefetchCount = _settings.PrefetchCount > 0 ? _settings.PrefetchCount : 1;
var messages = await _storageQueueClient.GetMessagesAsync <T>(prefetchCount, _settings.MessageLockTimeout).ConfigureAwait(false);
messagesFound = messages.Any();
_log.Debug("Prefetched {MessageCount} messages from {QueueName} in {Name}", messages.Count, queueName, nameof(StorageQueueMessagePump));
foreach (var message in messages)
{
var cts = new CancellationTokenSource(_settings.MessageLockTimeout);
try
{
_log.Debug("Processing {@Message} in {Name}", message, nameof(StorageQueueMessagePump));
_log.Debug("{ThreadCount} remaining threads that can process messages in {QueueName} in {Name}", _maxConcurrent.CurrentCount, queueName, nameof(StorageQueueMessagePump));
await _maxConcurrent.WaitAsync(cts.Token).ConfigureAwait(false);
}
catch (OperationCanceledException operationCanceledException)
{
_log.Debug(operationCanceledException, "Operation canceled for {@Message} in {QueueName} in {Name}", message, queueName, nameof(StorageQueueMessagePump));
//If we are still waiting when the message has not been scheduled for execution timeouts
continue;
}
#pragma warning disable 4014 //No need to await the result, let's keep the pump going
Task.Run(async() => await action.Invoke(message, cts.Token).ConfigureAwait(false), cts.Token).ContinueWith(task => _maxConcurrent.Release()).ConfigureAwait(false);
#pragma warning restore 4014
}
}
catch (Exception e)
{
_log.Error(e, "StorageQueueMessagePump error in {MessageType}", typeof(T));
}
finally
{
if (!messagesFound)
{
//Only delay pump if no messages were found
await Task.Delay(_pollingInterval).ConfigureAwait(false);
}
}
}
public async Task ProcessAsync <T>(IMessageStateHandler <T> messageStateHandler, IPipelineInformation pipelineInformation, IMessageProcessor next, CancellationToken cancellationToken) where T : class, IMessage
{
var queueName = AutoMessageMapper.GetQueueName <T>();
var log = pipelineInformation?.HostConfiguration?.Log;
log?.Debug("{ThreadCount} remaining threads that can process messages in {QueueName} in {Name}", _semaphoreQueue.CurrentCount, queueName, nameof(ThrottlingMiddleware));
await _semaphoreQueue.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
await next.ProcessAsync(messageStateHandler, cancellationToken).ConfigureAwait(false);
}
finally
{
_semaphoreQueue.Release();
}
}
public async Task Should_not_attach_attachments_for_other_commands()
{
//arrange
var message = new TestCommand();
var nextProcessor = new Mock <IMessageProcessor>();
var stateHandler = new Mock <IMessageStateHandler <TestCommand> >();
stateHandler.Setup(x => x.GetMessageAsync()).ReturnsAsync(message);
var attachmentProvider = new Mock <IMessageAttachmentProvider>();
attachmentProvider.Setup(x => x.GetAttachmentAsync(AutoMessageMapper.GetQueueName <TestCommand>(), It.IsAny <string>(), It.IsAny <CancellationToken>()))
.ReturnsAsync(default(IMessageAttachment));
var middleware = new AttachmentMiddleware(attachmentProvider.Object);
//act
await middleware.ProcessAsync(stateHandler.Object, Mock.Of <IPipelineInformation>(), nextProcessor.Object, CancellationToken.None);
//assert
attachmentProvider.Verify(x => x.GetAttachmentAsync(It.IsAny <string>(), It.IsAny <string>(), It.IsAny <CancellationToken>()), Times.Never);
nextProcessor.Verify(x => x.ProcessAsync(stateHandler.Object, CancellationToken.None), Times.Once);
}
public async Task Should_delete_attachment_when_finished()
{
//arrange
var message = new AttachmentCommand();
var nextProcessor = new Mock <IMessageProcessor>();
var stream = new MemoryStream(Encoding.UTF8.GetBytes("this is a stream"));
var attachment = new MessageAttachment("test.txt", "text/plain", stream);
var stateHandler = new Mock <IMessageStateHandler <AttachmentCommand> >();
stateHandler.Setup(x => x.GetMessageAsync()).ReturnsAsync(message);
stateHandler.Setup(x => x.MessageProperties).Returns(new Dictionary <string, string> {
{ AttachmentUtility.AttachmentKey, "89BDF3DB-896C-448D-A84E-872CBA8DBC9F" }
});
var attachmentProvider = new Mock <IMessageAttachmentProvider>();
attachmentProvider.Setup(x => x.GetAttachmentAsync(AutoMessageMapper.GetQueueName <AttachmentCommand>(), It.IsAny <string>(), It.IsAny <CancellationToken>()))
.ReturnsAsync(attachment);
var middleware = new AttachmentMiddleware(attachmentProvider.Object);
//act
await middleware.ProcessAsync(stateHandler.Object, nextProcessor.Object, CancellationToken.None);
//assert
attachmentProvider.Verify(x => x.DeleteAttachmentAsync(AutoMessageMapper.GetQueueName <AttachmentCommand>(), It.IsAny <string>(), It.IsAny <CancellationToken>()), Times.Once);
}
private IStorageQueueClient GetClient <T>() where T : class, ICommand
{
return(_queueClients.GetOrAdd(typeof(T), type => new StorageQueueClient(_options, AutoMessageMapper.GetQueueName <T>())));
}
public Task SendAsync <T>(T message) where T : IRedisCommand
{
var queueName = AutoMessageMapper.GetQueueName <T>();
return(SendAsync(message, queueName));
}
public RedisCommandChannelReceiver(IConnectionMultiplexer connectionMultiplexer, IProcessingSettings settings, IMessageSerializer serializer, RedisConfiguration configuration, IHostConfiguration hostConfiguration, IMessageProcessor processor)
: base(connectionMultiplexer, AutoMessageMapper.GetQueueName <T>(), settings, serializer, configuration, hostConfiguration, processor)
{
}
public Task SendAsync <T>(IEnumerable <T> messages) where T : IRedisCommand
{
var queueName = AutoMessageMapper.GetQueueName <T>();
return(SendAsync <T>(messages.ToList(), queueName));
}
private SubscriptionClient CreateSubscriptionClient <T>(string subscriptionName) where T : IEvent
{
var topicName = AutoMessageMapper.GetQueueName <T>();
return(new SubscriptionClient(_connectionString, topicName, subscriptionName, ReceiveMode.PeekLock, RetryPolicy.Default));
}
private TopicClient CreateTopicClient <T>() where T : IEvent
{
var topicName = AutoMessageMapper.GetQueueName <T>();
return(new TopicClient(_connectionString, topicName, RetryPolicy.Default));
}
private QueueClient CreateQueueClient <T>() where T : ICommand
{
var queueName = AutoMessageMapper.GetQueueName <T>();
return(new QueueClient(_connectionString, queueName, ReceiveMode.PeekLock, RetryPolicy.Default));
}
internal async Task PumpAsync <T>(Func <StorageQueueMessage, CancellationToken, Task> action, CancellationToken cancellationToken) where T : IStorageQueueCommand
{
var messagesFound = false;
try
{
var queueName = AutoMessageMapper.GetQueueName <T>();
var prefetchCount = _settings.PrefetchCount > 0 ? _settings.PrefetchCount : 1;
TimeSpan visibilityTimeout;
if (_settings is IExtendMessageLockTimeout extendMessageLockTimeout)
{
visibilityTimeout = extendMessageLockTimeout.ExtensionDuration;
}
else
{
visibilityTimeout = _settings.MessageLockTimeout;
}
//Make sure the lock still exist when the process is cancelled by token, otherwise the message cannot be abandoned
visibilityTimeout += TimeSpan.FromMinutes(2);
var messages = await _storageQueueClient.GetMessagesAsync <T>(prefetchCount, visibilityTimeout).ConfigureAwait(false);
messagesFound = messages.Any();
_log.Debug("Prefetched {MessageCount} messages from {QueueName} in {Name}", messages.Count, queueName, nameof(StorageQueueMessagePump));
foreach (var message in messages)
{
var timeoutToken = new CancellationTokenSource(_settings.MessageLockTimeout);
var linkedToken = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, timeoutToken.Token);
try
{
_log.Debug("Processing {@Message} in {Name}", message, nameof(StorageQueueMessagePump));
_log.Debug("{ThreadCount} remaining threads that can process messages in {QueueName} in {Name}", _maxConcurrent.CurrentCount, queueName, nameof(StorageQueueMessagePump));
await _maxConcurrent.WaitAsync(timeoutToken.Token).ConfigureAwait(false);
}
catch (OperationCanceledException operationCanceledException)
{
_log.Debug(operationCanceledException, "Operation canceled for {@Message} in {QueueName} in {Name}", message, queueName, nameof(StorageQueueMessagePump));
//If we are still waiting when the message has not been scheduled for execution timeouts
continue;
}
#pragma warning disable 4014 //No need to await the result, let's keep the pump going
Task.Run(async() => await action.Invoke(message, linkedToken.Token).ConfigureAwait(false), timeoutToken.Token)
.ContinueWith(task =>
{
_maxConcurrent.Release();
timeoutToken.Dispose();
linkedToken.Dispose();
}).ConfigureAwait(false);
#pragma warning restore 4014
}
}
catch (Exception e)
{
_log.Error(e, "StorageQueueMessagePump error in {MessageType}", typeof(T));
}
finally
{
if (!messagesFound)
{
//Only delay pump if no messages were found
await Task.Delay(_pollingInterval).ConfigureAwait(false);
}
}
}
public RedisEventChannelReceiver(IConnectionMultiplexer connectionMultiplexer, IProcessingSettings settings, IMessageSerializer serializer, IEventSubscription <T> subscription, RedisConfiguration configuration, IHostConfiguration hostConfiguration, IMessageProcessor processor)
: base(connectionMultiplexer, RedisQueueConventions.GetSubscriptionQueueName(AutoMessageMapper.GetQueueName <T>(), subscription.Name), settings, serializer, configuration, hostConfiguration, processor)
{
_subscription = subscription;
_redisConfiguration = configuration;
}
public void Should_find_registered_generic()
{
var name = AutoMessageMapper.GetQueueName <RegisteredCommand>();
name.Should().Be("queue");
}
public void Should_find_registered_type()
{
var name = AutoMessageMapper.GetQueueName(typeof(RegisteredCommand));
name.Should().Be("queue");
}