private void CreateSubscriptionListener(SqsQueueBase queue, string messageTypeName) { var sqsSubscriptionListener = new SqsNotificationListener(queue, Bus.SerialisationRegister, Bus.Monitor, _subscriptionConfig.OnError, Bus.MessageLock); Bus.AddNotificationTopicSubscriber(messageTypeName, sqsSubscriptionListener); if (_subscriptionConfig.MaxAllowedMessagesInFlight.HasValue) { sqsSubscriptionListener.WithMaximumConcurrentLimitOnMessagesInFlightOf(_subscriptionConfig.MaxAllowedMessagesInFlight.Value); } if (_subscriptionConfig.MessageProcessingStrategy != null) { sqsSubscriptionListener.WithMessageProcessingStrategy(_subscriptionConfig.MessageProcessingStrategy); } }
private void CreateSubscriptionListener <T>(string region, SqsQueueBase queue) where T : Message { var sqsSubscriptionListener = new SqsNotificationListener(queue, Bus.SerialisationRegister, Bus.Monitor, _subscriptionConfig.OnError, Bus.MessageLock); sqsSubscriptionListener.Subscribers.Add(new Subscriber(typeof(T))); Bus.AddNotificationSubscriber(region, sqsSubscriptionListener); if (_subscriptionConfig.MaxAllowedMessagesInFlight.HasValue) { sqsSubscriptionListener.WithMaximumConcurrentLimitOnMessagesInFlightOf(_subscriptionConfig.MaxAllowedMessagesInFlight.Value); } if (_subscriptionConfig.MessageProcessingStrategy != null) { sqsSubscriptionListener.WithMessageProcessingStrategy(_subscriptionConfig.MessageProcessingStrategy); } }
/// <summary> /// Set message handlers for the given topic /// </summary> /// <typeparam name="T">Message type to be handled</typeparam> /// <param name="handler">Handler for the message type</param> /// <returns></returns> public IHaveFulfilledSubscriptionRequirements WithMessageHandler <T>(IHandler <T> handler) where T : Message { _subscriptionConfig.Topic = typeof(T).Name.ToLower(); var publishEndpointProvider = CreatePublisherEndpointProvider(_subscriptionConfig); _subscriptionConfig.PublishEndpoint = publishEndpointProvider.GetLocationName(); _subscriptionConfig.Validate(); var subscriptionEndpointProvider = CreateSubscriptiuonEndpointProvider(_subscriptionConfig); _subscriptionConfig.QueueName = subscriptionEndpointProvider.GetLocationName(); var queue = _amazonQueueCreator.VerifyOrCreateQueue(Bus.Config.Region, Bus.SerialisationRegister, _subscriptionConfig); var sqsSubscriptionListener = new SqsNotificationListener(queue, Bus.SerialisationRegister, Bus.Monitor, _subscriptionConfig.OnError, Bus.MessageLock); Bus.AddNotificationTopicSubscriber(_subscriptionConfig.Topic, sqsSubscriptionListener); if (_subscriptionConfig.MaxAllowedMessagesInFlight.HasValue) { sqsSubscriptionListener.WithMaximumConcurrentLimitOnMessagesInFlightOf(_subscriptionConfig.MaxAllowedMessagesInFlight.Value); } if (_subscriptionConfig.MessageProcessingStrategy != null) { sqsSubscriptionListener.WithMessageProcessingStrategy(_subscriptionConfig.MessageProcessingStrategy); } Log.Info(string.Format("Created SQS topic subscription - Topic: {0}, QueueName: {1}", _subscriptionConfig.Topic, _subscriptionConfig.QueueName)); _subscriptionConfigured = true; if (!_subscriptionConfigured) { ConfigureSubscriptionWith(conf => conf.ErrorQueueOptOut = _subscriptionConfig.ErrorQueueOptOut); } Bus.SerialisationRegister.AddSerialiser <T>(_serialisationFactory.GetSerialiser <T>()); Bus.AddMessageHandler(handler); Log.Info(string.Format("Added a message handler - Topic: {0}, MessageType: {1}, HandlerName: {2}", _subscriptionConfig.Topic, typeof(T).Name, handler.GetType().Name)); return(this); }
public async Task HandlingManyMessages(int throttleMessageCount) { // Arrange var fixture = new JustSayingFixture(OutputHelper); var client = fixture.CreateSqsClient(); var queue = new SqsQueueByName(fixture.Region, fixture.UniqueName, client, 1, fixture.LoggerFactory); if (!await queue.ExistsAsync()) { await queue.CreateAsync(new SqsBasicConfiguration()); if (!fixture.IsSimulator) { // Wait for up to 60 secs for queue creation to be guaranteed completed by AWS using (var cts = new CancellationTokenSource(TimeSpan.FromMinutes(1))) { while (!cts.IsCancellationRequested) { await Task.Delay(TimeSpan.FromSeconds(2)); if (await queue.ExistsAsync()) { break; } } } } } Assert.True(await queue.ExistsAsync(), "The queue was not created."); OutputHelper.WriteLine($"{DateTime.Now} - Adding {throttleMessageCount} messages to the queue."); var entriesAdded = 0; // Add some messages do { var entries = new List <SendMessageBatchRequestEntry>(); for (var j = 0; j < 10; j++) { var batchEntry = new SendMessageBatchRequestEntry { MessageBody = $"{{\"Subject\":\"GenericMessage\", \"Message\": \"{entriesAdded}\"}}", Id = Guid.NewGuid().ToString() }; entries.Add(batchEntry); entriesAdded++; } await client.SendMessageBatchAsync(queue.Url, entries); }while (entriesAdded < throttleMessageCount); OutputHelper.WriteLine($"{DateTime.Now} - Done adding messages."); var handleCount = 0; var serialisations = Substitute.For <IMessageSerialisationRegister>(); var monitor = Substitute.For <IMessageMonitor>(); var handler = Substitute.For <IHandlerAsync <SimpleMessage> >(); handler.Handle(null).ReturnsForAnyArgs(true).AndDoes(_ => Interlocked.Increment(ref handleCount)); serialisations.DeserializeMessage(string.Empty).ReturnsForAnyArgs(new SimpleMessage()); var listener = new SqsNotificationListener(queue, serialisations, monitor, fixture.LoggerFactory); listener.AddMessageHandler(() => handler); // Act var stopwatch = Stopwatch.StartNew(); listener.Listen(); using (var cts = new CancellationTokenSource(TimeSpan.FromMinutes(5))) { do { if (!fixture.IsSimulator) { await Task.Delay(TimeSpan.FromSeconds(5)); } OutputHelper.WriteLine($"{DateTime.Now} - Handled {handleCount} messages. Waiting for completion."); }while (handleCount < throttleMessageCount && !cts.IsCancellationRequested); } listener.StopListening(); stopwatch.Stop(); OutputHelper.WriteLine($"{DateTime.Now} - Handled {handleCount:N0} messages."); OutputHelper.WriteLine($"{DateTime.Now} - Took {stopwatch.ElapsedMilliseconds} ms"); OutputHelper.WriteLine($"{DateTime.Now} - Throughput {(float)handleCount / stopwatch.ElapsedMilliseconds * 1000} messages/second"); // Assert Assert.Equal(throttleMessageCount, handleCount); }
// Use this to manually test the performance / throttling of getting messages out of the queue. public async Task HandlingManyMessages(int throttleMessageCount) { var locker = new object(); var awsQueueClient = CreateMeABus.DefaultClientFactory().GetSqsClient(RegionEndpoint.EUWest1); var q = new SqsQueueByName(RegionEndpoint.EUWest1, "throttle_test", awsQueueClient, 1, new LoggerFactory()); if (!await q.ExistsAsync()) { await q.CreateAsync(new SqsBasicConfiguration()); await Task.Delay(TimeSpan.FromMinutes(1)); // wait 60 secs for queue creation to be guaranteed completed by aws. :( } Assert.True(await q.ExistsAsync()); Console.WriteLine($"{DateTime.Now} - Adding {throttleMessageCount} messages to the queue."); var entriesAdded = 0; // Add some messages do { var entries = new List <SendMessageBatchRequestEntry>(); for (var j = 0; j < 10; j++) { var batchEntry = new SendMessageBatchRequestEntry { MessageBody = "{\"Subject\":\"GenericMessage\", \"Message\": \"" + entriesAdded.ToString() + "\"}", Id = Guid.NewGuid().ToString() }; entries.Add(batchEntry); entriesAdded++; } await awsQueueClient.SendMessageBatchAsync(new SendMessageBatchRequest { QueueUrl = q.Url, Entries = entries }); }while (entriesAdded < throttleMessageCount); Console.WriteLine($"{DateTime.Now} - Done adding messages."); var handleCount = 0; var serialisations = Substitute.For <IMessageSerialisationRegister>(); var monitor = Substitute.For <IMessageMonitor>(); var handler = Substitute.For <IHandlerAsync <GenericMessage> >(); handler.Handle(null).ReturnsForAnyArgs(true).AndDoes(x => { lock (locker) { handleCount++; } }); serialisations.DeserializeMessage(string.Empty).ReturnsForAnyArgs(new GenericMessage()); var listener = new SqsNotificationListener(q, serialisations, monitor, new LoggerFactory()); listener.AddMessageHandler(() => handler); var stopwatch = new Stopwatch(); stopwatch.Start(); listener.Listen(); var waitCount = 0; do { await Task.Delay(TimeSpan.FromSeconds(5)); Console.WriteLine($"{DateTime.Now} - Handled {handleCount} messages. Waiting for completion."); waitCount++; }while (handleCount < throttleMessageCount && waitCount < 100); listener.StopListening(); stopwatch.Stop(); Console.WriteLine($"{DateTime.Now} - Handled {handleCount} messages."); Console.WriteLine($"{DateTime.Now} - Took {stopwatch.ElapsedMilliseconds} ms"); Console.WriteLine( $"{DateTime.Now} - Throughput {(float) handleCount/stopwatch.ElapsedMilliseconds*1000} msg/sec"); Assert.Equal(throttleMessageCount, handleCount); }
// Use this to manually test the performance / throttling of getting messages out of the queue. public void HandlingManyMessages(int throttleMessageCount) { var locker = new object(); var awsQueueClient = CreateMeABus.DefaultClientFactory().GetSqsClient(RegionEndpoint.EUWest1); var q = new SqsQueueByName(RegionEndpoint.EUWest1, "throttle_test", awsQueueClient, 1); if (!q.Exists()) { q.Create(new SqsBasicConfiguration()); Thread.Sleep(TimeSpan.FromMinutes(1)); // wait 60 secs for queue creation to be guaranteed completed by aws. :( } Assert.True(q.Exists()); Console.WriteLine("{0} - Adding {1} messages to the queue.", DateTime.Now, throttleMessageCount); var entriesAdded = 0; // Add some messages do { var entries = new List<SendMessageBatchRequestEntry>(); for (var j = 0; j < 10; j++) { var batchEntry = new SendMessageBatchRequestEntry { MessageBody = "{\"Subject\":\"GenericMessage\", \"Message\": \"" + entriesAdded.ToString() + "\"}", Id = Guid.NewGuid().ToString() }; entries.Add(batchEntry); entriesAdded++; } awsQueueClient.SendMessageBatch(new SendMessageBatchRequest { QueueUrl = q.Url, Entries = entries }); } while (entriesAdded < throttleMessageCount); Console.WriteLine("{0} - Done adding messages.", DateTime.Now); var handleCount = 0; var serialisations = Substitute.For<IMessageSerialisationRegister>(); var monitor = Substitute.For<IMessageMonitor>(); var handler = Substitute.For<IHandler<GenericMessage>>(); handler.Handle(null).ReturnsForAnyArgs(true).AndDoes(x => {lock (locker) { handleCount++; } }); serialisations.DeserializeMessage(string.Empty).ReturnsForAnyArgs(new GenericMessage()); var listener = new SqsNotificationListener(q, serialisations, monitor); listener.AddMessageHandler(() => handler); var stopwatch = new Stopwatch(); stopwatch.Start(); listener.Listen(); var waitCount = 0; do { Thread.Sleep(TimeSpan.FromSeconds(5)); Console.WriteLine("{0} - Handled {1} messages. Waiting for completion.", DateTime.Now, handleCount); waitCount++; } while (handleCount < throttleMessageCount && waitCount < 100); listener.StopListening(); stopwatch.Stop(); Console.WriteLine("{0} - Handled {1} messages.", DateTime.Now, handleCount); Console.WriteLine("{0} - Took {1} ms", DateTime.Now, stopwatch.ElapsedMilliseconds); Console.WriteLine("{0} - Throughput {1} msg/sec", DateTime.Now, (float)handleCount / stopwatch.ElapsedMilliseconds * 1000); Assert.AreEqual(throttleMessageCount, handleCount); }
// Use this to manually test the performance / throttling of getting messages out of the queue. public void HandlingManyMessages(int throttleMessageCount) { var locker = new object(); var awsQueueClient = AWSClientFactory.CreateAmazonSQSClient(RegionEndpoint.EUWest1); var q = new SqsQueueByName("throttle_test", awsQueueClient, 1); if (!q.Exists()) { q.Create(new SqsBasicConfiguration()); Thread.Sleep(TimeSpan.FromMinutes(1)); // wait 60 secs for queue creation to be guaranteed completed by aws. :( } Assert.True(q.Exists()); Console.WriteLine("{0} - Adding {1} messages to the queue.", DateTime.Now, throttleMessageCount); var entriesAdded = 0; // Add some messages do { var entries = new List <SendMessageBatchRequestEntry>(); for (var j = 0; j < 10; j++) { var batchEntry = new SendMessageBatchRequestEntry { MessageBody = "{\"Subject\":\"GenericMessage\", \"Message\": \"" + entriesAdded.ToString() + "\"}", Id = Guid.NewGuid().ToString() }; entries.Add(batchEntry); entriesAdded++; } awsQueueClient.SendMessageBatch(new SendMessageBatchRequest { QueueUrl = q.Url, Entries = entries }); }while (entriesAdded < throttleMessageCount); Console.WriteLine("{0} - Done adding messages.", DateTime.Now); var handleCount = 0; var serialisations = Substitute.For <IMessageSerialisationRegister>(); var monitor = Substitute.For <IMessageMonitor>(); var handler = Substitute.For <IHandler <GenericMessage> >(); handler.Handle(null).ReturnsForAnyArgs(true).AndDoes(x => { lock (locker) { Thread.Sleep(10); handleCount++; } }); var serialiser = Substitute.For <IMessageSerialiser <GenericMessage> >(); serialiser.Deserialise(string.Empty).ReturnsForAnyArgs(new GenericMessage()); serialisations.GetSerialiser(string.Empty).ReturnsForAnyArgs(serialiser); var listener = new SqsNotificationListener(q, serialisations, monitor); listener.AddMessageHandler(handler); listener.Listen(); var waitCount = 0; do { Thread.Sleep(TimeSpan.FromSeconds(5)); Console.WriteLine("{0} - Handled {1} messages. Waiting for completion.", DateTime.Now, handleCount); waitCount++; }while (handleCount < throttleMessageCount && waitCount < 100); listener.StopListening(); Console.WriteLine("{0} - Handled {1} messages.", DateTime.Now, handleCount); Assert.AreEqual(throttleMessageCount, handleCount); }