private void CreateSubscriptionListener(SqsQueueBase queue, string messageTypeName)
{
var sqsSubscriptionListener = new SqsNotificationListener(queue, Bus.SerialisationRegister, Bus.Monitor, _subscriptionConfig.OnError, Bus.MessageLock);
Bus.AddNotificationTopicSubscriber(messageTypeName, sqsSubscriptionListener);
if (_subscriptionConfig.MaxAllowedMessagesInFlight.HasValue)
{
sqsSubscriptionListener.WithMaximumConcurrentLimitOnMessagesInFlightOf(_subscriptionConfig.MaxAllowedMessagesInFlight.Value);
}
if (_subscriptionConfig.MessageProcessingStrategy != null)
{
sqsSubscriptionListener.WithMessageProcessingStrategy(_subscriptionConfig.MessageProcessingStrategy);
}
}
private void CreateSubscriptionListener <T>(string region, SqsQueueBase queue) where T : Message
{
var sqsSubscriptionListener = new SqsNotificationListener(queue, Bus.SerialisationRegister, Bus.Monitor, _subscriptionConfig.OnError, Bus.MessageLock);
sqsSubscriptionListener.Subscribers.Add(new Subscriber(typeof(T)));
Bus.AddNotificationSubscriber(region, sqsSubscriptionListener);
if (_subscriptionConfig.MaxAllowedMessagesInFlight.HasValue)
{
sqsSubscriptionListener.WithMaximumConcurrentLimitOnMessagesInFlightOf(_subscriptionConfig.MaxAllowedMessagesInFlight.Value);
}
if (_subscriptionConfig.MessageProcessingStrategy != null)
{
sqsSubscriptionListener.WithMessageProcessingStrategy(_subscriptionConfig.MessageProcessingStrategy);
}
}
/// <summary>
/// Set message handlers for the given topic
/// </summary>
/// <typeparam name="T">Message type to be handled</typeparam>
/// <param name="handler">Handler for the message type</param>
/// <returns></returns>
public IHaveFulfilledSubscriptionRequirements WithMessageHandler <T>(IHandler <T> handler) where T : Message
{
_subscriptionConfig.Topic = typeof(T).Name.ToLower();
var publishEndpointProvider = CreatePublisherEndpointProvider(_subscriptionConfig);
_subscriptionConfig.PublishEndpoint = publishEndpointProvider.GetLocationName();
_subscriptionConfig.Validate();
var subscriptionEndpointProvider = CreateSubscriptiuonEndpointProvider(_subscriptionConfig);
_subscriptionConfig.QueueName = subscriptionEndpointProvider.GetLocationName();
var queue = _amazonQueueCreator.VerifyOrCreateQueue(Bus.Config.Region, Bus.SerialisationRegister, _subscriptionConfig);
var sqsSubscriptionListener = new SqsNotificationListener(queue, Bus.SerialisationRegister, Bus.Monitor, _subscriptionConfig.OnError, Bus.MessageLock);
Bus.AddNotificationTopicSubscriber(_subscriptionConfig.Topic, sqsSubscriptionListener);
if (_subscriptionConfig.MaxAllowedMessagesInFlight.HasValue)
{
sqsSubscriptionListener.WithMaximumConcurrentLimitOnMessagesInFlightOf(_subscriptionConfig.MaxAllowedMessagesInFlight.Value);
}
if (_subscriptionConfig.MessageProcessingStrategy != null)
{
sqsSubscriptionListener.WithMessageProcessingStrategy(_subscriptionConfig.MessageProcessingStrategy);
}
Log.Info(string.Format("Created SQS topic subscription - Topic: {0}, QueueName: {1}", _subscriptionConfig.Topic, _subscriptionConfig.QueueName));
_subscriptionConfigured = true;
if (!_subscriptionConfigured)
{
ConfigureSubscriptionWith(conf => conf.ErrorQueueOptOut = _subscriptionConfig.ErrorQueueOptOut);
}
Bus.SerialisationRegister.AddSerialiser <T>(_serialisationFactory.GetSerialiser <T>());
Bus.AddMessageHandler(handler);
Log.Info(string.Format("Added a message handler - Topic: {0}, MessageType: {1}, HandlerName: {2}", _subscriptionConfig.Topic, typeof(T).Name, handler.GetType().Name));
return(this);
}
public async Task HandlingManyMessages(int throttleMessageCount)
{
// Arrange
var fixture = new JustSayingFixture(OutputHelper);
var client = fixture.CreateSqsClient();
var queue = new SqsQueueByName(fixture.Region, fixture.UniqueName, client, 1, fixture.LoggerFactory);
if (!await queue.ExistsAsync())
{
await queue.CreateAsync(new SqsBasicConfiguration());
if (!fixture.IsSimulator)
{
// Wait for up to 60 secs for queue creation to be guaranteed completed by AWS
using (var cts = new CancellationTokenSource(TimeSpan.FromMinutes(1)))
{
while (!cts.IsCancellationRequested)
{
await Task.Delay(TimeSpan.FromSeconds(2));
if (await queue.ExistsAsync())
{
break;
}
}
}
}
}
Assert.True(await queue.ExistsAsync(), "The queue was not created.");
OutputHelper.WriteLine($"{DateTime.Now} - Adding {throttleMessageCount} messages to the queue.");
var entriesAdded = 0;
// Add some messages
do
{
var entries = new List <SendMessageBatchRequestEntry>();
for (var j = 0; j < 10; j++)
{
var batchEntry = new SendMessageBatchRequestEntry
{
MessageBody = $"{{\"Subject\":\"GenericMessage\", \"Message\": \"{entriesAdded}\"}}",
Id = Guid.NewGuid().ToString()
};
entries.Add(batchEntry);
entriesAdded++;
}
await client.SendMessageBatchAsync(queue.Url, entries);
}while (entriesAdded < throttleMessageCount);
OutputHelper.WriteLine($"{DateTime.Now} - Done adding messages.");
var handleCount = 0;
var serialisations = Substitute.For <IMessageSerialisationRegister>();
var monitor = Substitute.For <IMessageMonitor>();
var handler = Substitute.For <IHandlerAsync <SimpleMessage> >();
handler.Handle(null).ReturnsForAnyArgs(true).AndDoes(_ => Interlocked.Increment(ref handleCount));
serialisations.DeserializeMessage(string.Empty).ReturnsForAnyArgs(new SimpleMessage());
var listener = new SqsNotificationListener(queue, serialisations, monitor, fixture.LoggerFactory);
listener.AddMessageHandler(() => handler);
// Act
var stopwatch = Stopwatch.StartNew();
listener.Listen();
using (var cts = new CancellationTokenSource(TimeSpan.FromMinutes(5)))
{
do
{
if (!fixture.IsSimulator)
{
await Task.Delay(TimeSpan.FromSeconds(5));
}
OutputHelper.WriteLine($"{DateTime.Now} - Handled {handleCount} messages. Waiting for completion.");
}while (handleCount < throttleMessageCount && !cts.IsCancellationRequested);
}
listener.StopListening();
stopwatch.Stop();
OutputHelper.WriteLine($"{DateTime.Now} - Handled {handleCount:N0} messages.");
OutputHelper.WriteLine($"{DateTime.Now} - Took {stopwatch.ElapsedMilliseconds} ms");
OutputHelper.WriteLine($"{DateTime.Now} - Throughput {(float)handleCount / stopwatch.ElapsedMilliseconds * 1000} messages/second");
// Assert
Assert.Equal(throttleMessageCount, handleCount);
}
// Use this to manually test the performance / throttling of getting messages out of the queue.
public async Task HandlingManyMessages(int throttleMessageCount)
{
var locker = new object();
var awsQueueClient = CreateMeABus.DefaultClientFactory().GetSqsClient(RegionEndpoint.EUWest1);
var q = new SqsQueueByName(RegionEndpoint.EUWest1, "throttle_test", awsQueueClient, 1, new LoggerFactory());
if (!await q.ExistsAsync())
{
await q.CreateAsync(new SqsBasicConfiguration());
await Task.Delay(TimeSpan.FromMinutes(1)); // wait 60 secs for queue creation to be guaranteed completed by aws. :(
}
Assert.True(await q.ExistsAsync());
Console.WriteLine($"{DateTime.Now} - Adding {throttleMessageCount} messages to the queue.");
var entriesAdded = 0;
// Add some messages
do
{
var entries = new List <SendMessageBatchRequestEntry>();
for (var j = 0; j < 10; j++)
{
var batchEntry = new SendMessageBatchRequestEntry
{
MessageBody = "{\"Subject\":\"GenericMessage\", \"Message\": \"" + entriesAdded.ToString() + "\"}",
Id = Guid.NewGuid().ToString()
};
entries.Add(batchEntry);
entriesAdded++;
}
await awsQueueClient.SendMessageBatchAsync(new SendMessageBatchRequest { QueueUrl = q.Url, Entries = entries });
}while (entriesAdded < throttleMessageCount);
Console.WriteLine($"{DateTime.Now} - Done adding messages.");
var handleCount = 0;
var serialisations = Substitute.For <IMessageSerialisationRegister>();
var monitor = Substitute.For <IMessageMonitor>();
var handler = Substitute.For <IHandlerAsync <GenericMessage> >();
handler.Handle(null).ReturnsForAnyArgs(true).AndDoes(x => { lock (locker) { handleCount++; } });
serialisations.DeserializeMessage(string.Empty).ReturnsForAnyArgs(new GenericMessage());
var listener = new SqsNotificationListener(q, serialisations, monitor, new LoggerFactory());
listener.AddMessageHandler(() => handler);
var stopwatch = new Stopwatch();
stopwatch.Start();
listener.Listen();
var waitCount = 0;
do
{
await Task.Delay(TimeSpan.FromSeconds(5));
Console.WriteLine($"{DateTime.Now} - Handled {handleCount} messages. Waiting for completion.");
waitCount++;
}while (handleCount < throttleMessageCount && waitCount < 100);
listener.StopListening();
stopwatch.Stop();
Console.WriteLine($"{DateTime.Now} - Handled {handleCount} messages.");
Console.WriteLine($"{DateTime.Now} - Took {stopwatch.ElapsedMilliseconds} ms");
Console.WriteLine(
$"{DateTime.Now} - Throughput {(float) handleCount/stopwatch.ElapsedMilliseconds*1000} msg/sec");
Assert.Equal(throttleMessageCount, handleCount);
}
// Use this to manually test the performance / throttling of getting messages out of the queue.
public void HandlingManyMessages(int throttleMessageCount)
{
var locker = new object();
var awsQueueClient = CreateMeABus.DefaultClientFactory().GetSqsClient(RegionEndpoint.EUWest1);
var q = new SqsQueueByName(RegionEndpoint.EUWest1, "throttle_test", awsQueueClient, 1);
if (!q.Exists())
{
q.Create(new SqsBasicConfiguration());
Thread.Sleep(TimeSpan.FromMinutes(1)); // wait 60 secs for queue creation to be guaranteed completed by aws. :(
}
Assert.True(q.Exists());
Console.WriteLine("{0} - Adding {1} messages to the queue.", DateTime.Now, throttleMessageCount);
var entriesAdded = 0;
// Add some messages
do
{
var entries = new List<SendMessageBatchRequestEntry>();
for (var j = 0; j < 10; j++)
{
var batchEntry = new SendMessageBatchRequestEntry
{
MessageBody = "{\"Subject\":\"GenericMessage\", \"Message\": \"" + entriesAdded.ToString() + "\"}",
Id = Guid.NewGuid().ToString()
};
entries.Add(batchEntry);
entriesAdded++;
}
awsQueueClient.SendMessageBatch(new SendMessageBatchRequest { QueueUrl = q.Url, Entries = entries });
}
while (entriesAdded < throttleMessageCount);
Console.WriteLine("{0} - Done adding messages.", DateTime.Now);
var handleCount = 0;
var serialisations = Substitute.For<IMessageSerialisationRegister>();
var monitor = Substitute.For<IMessageMonitor>();
var handler = Substitute.For<IHandler<GenericMessage>>();
handler.Handle(null).ReturnsForAnyArgs(true).AndDoes(x => {lock (locker) { handleCount++; } });
serialisations.DeserializeMessage(string.Empty).ReturnsForAnyArgs(new GenericMessage());
var listener = new SqsNotificationListener(q, serialisations, monitor);
listener.AddMessageHandler(() => handler);
var stopwatch = new Stopwatch();
stopwatch.Start();
listener.Listen();
var waitCount = 0;
do
{
Thread.Sleep(TimeSpan.FromSeconds(5));
Console.WriteLine("{0} - Handled {1} messages. Waiting for completion.", DateTime.Now, handleCount);
waitCount++;
}
while (handleCount < throttleMessageCount && waitCount < 100);
listener.StopListening();
stopwatch.Stop();
Console.WriteLine("{0} - Handled {1} messages.", DateTime.Now, handleCount);
Console.WriteLine("{0} - Took {1} ms", DateTime.Now, stopwatch.ElapsedMilliseconds);
Console.WriteLine("{0} - Throughput {1} msg/sec", DateTime.Now, (float)handleCount / stopwatch.ElapsedMilliseconds * 1000);
Assert.AreEqual(throttleMessageCount, handleCount);
}
// Use this to manually test the performance / throttling of getting messages out of the queue.
public void HandlingManyMessages(int throttleMessageCount)
{
var locker = new object();
var awsQueueClient = AWSClientFactory.CreateAmazonSQSClient(RegionEndpoint.EUWest1);
var q = new SqsQueueByName("throttle_test", awsQueueClient, 1);
if (!q.Exists())
{
q.Create(new SqsBasicConfiguration());
Thread.Sleep(TimeSpan.FromMinutes(1)); // wait 60 secs for queue creation to be guaranteed completed by aws. :(
}
Assert.True(q.Exists());
Console.WriteLine("{0} - Adding {1} messages to the queue.", DateTime.Now, throttleMessageCount);
var entriesAdded = 0;
// Add some messages
do
{
var entries = new List <SendMessageBatchRequestEntry>();
for (var j = 0; j < 10; j++)
{
var batchEntry = new SendMessageBatchRequestEntry
{
MessageBody = "{\"Subject\":\"GenericMessage\", \"Message\": \"" + entriesAdded.ToString() + "\"}",
Id = Guid.NewGuid().ToString()
};
entries.Add(batchEntry);
entriesAdded++;
}
awsQueueClient.SendMessageBatch(new SendMessageBatchRequest {
QueueUrl = q.Url, Entries = entries
});
}while (entriesAdded < throttleMessageCount);
Console.WriteLine("{0} - Done adding messages.", DateTime.Now);
var handleCount = 0;
var serialisations = Substitute.For <IMessageSerialisationRegister>();
var monitor = Substitute.For <IMessageMonitor>();
var handler = Substitute.For <IHandler <GenericMessage> >();
handler.Handle(null).ReturnsForAnyArgs(true).AndDoes(x => { lock (locker) { Thread.Sleep(10); handleCount++; } });
var serialiser = Substitute.For <IMessageSerialiser <GenericMessage> >();
serialiser.Deserialise(string.Empty).ReturnsForAnyArgs(new GenericMessage());
serialisations.GetSerialiser(string.Empty).ReturnsForAnyArgs(serialiser);
var listener = new SqsNotificationListener(q, serialisations, monitor);
listener.AddMessageHandler(handler);
listener.Listen();
var waitCount = 0;
do
{
Thread.Sleep(TimeSpan.FromSeconds(5));
Console.WriteLine("{0} - Handled {1} messages. Waiting for completion.", DateTime.Now, handleCount);
waitCount++;
}while (handleCount < throttleMessageCount && waitCount < 100);
listener.StopListening();
Console.WriteLine("{0} - Handled {1} messages.", DateTime.Now, handleCount);
Assert.AreEqual(throttleMessageCount, handleCount);
}
