public async Task Sqs_Policy_Is_Applied_With_Wildcard() { // Arrange var handler = new ExactlyOnceHandlerWithTimeout(); var services = GivenJustSaying() .ConfigureJustSaying((builder) => builder.WithLoopbackTopic <TopicA>(UniqueName)) .ConfigureJustSaying((builder) => builder.WithLoopbackTopic <TopicB>(UniqueName)) .AddJustSayingHandler <TopicA, HandlerA>() .AddJustSayingHandler <TopicB, HandlerB>(); await WhenAsync( services, async (publisher, listener, serviceProvider, cancellationToken) => { listener.Start(cancellationToken); var clientFactory = serviceProvider.GetRequiredService <MessagingBusBuilder>().BuildClientFactory(); var loggerFactory = serviceProvider.GetRequiredService <ILoggerFactory>(); var client = clientFactory.GetSqsClient(Region); var queue = new SqsQueueByName(Region, UniqueName, client, 0, loggerFactory); await Patiently.AssertThatAsync(() => queue.ExistsAsync(), 60.Seconds()); dynamic policyJson = JObject.Parse(queue.Policy); policyJson.Statement.Count.ShouldBe(1, $"Expecting 1 statement in Sqs policy but found {policyJson.Statement.Count}."); }); }
private static async Task EnsureQueueExistsAsync(SqsQueueByName queue) { if (!await queue.ExistsAsync().ConfigureAwait(false)) { throw new InvalidOperationException($"{queue.QueueName} does not exist."); } }
public async Task QueueIsCreated() { async Task QueueIsCreatedInner() { var queue = new SqsQueueByName( TestFixture.Region, QueueName, Client, 0, TestFixture.LoggerFactory); await Patiently.AssertThatAsync( () => queue.ExistsAsync(), TimeSpan.FromSeconds(65)); } var task = QueueIsCreatedInner(); if (task == await Task.WhenAny(task, Task.Delay(TimeSpan.FromSeconds(70)))) { await task; } else { throw new TimeoutException(); } }
/// <inheritdoc /> void IPublicationBuilder <T> .Configure( JustSayingBus bus, IAwsClientFactoryProxy proxy, ILoggerFactory loggerFactory) { var logger = loggerFactory.CreateLogger <QueuePublicationBuilder <T> >(); logger.LogInformation("Adding SQS publisher for message type '{MessageType}'.", typeof(T)); var config = bus.Config; var region = config.Region ?? throw new InvalidOperationException($"Config cannot have a blank entry for the {nameof(config.Region)} property."); var writeConfiguration = new SqsWriteConfiguration(); ConfigureWrites?.Invoke(writeConfiguration); writeConfiguration.ApplyQueueNamingConvention <T>(config.QueueNamingConvention); bus.SerializationRegister.AddSerializer <T>(); var regionEndpoint = RegionEndpoint.GetBySystemName(region); var sqsClient = proxy.GetAwsClientFactory().GetSqsClient(regionEndpoint); var eventPublisher = new SqsMessagePublisher( sqsClient, bus.SerializationRegister, loggerFactory) { MessageResponseLogger = config.MessageResponseLogger }; #pragma warning disable 618 var sqsQueue = new SqsQueueByName( regionEndpoint, writeConfiguration.QueueName, sqsClient, writeConfiguration.RetryCountBeforeSendingToErrorQueue, loggerFactory); #pragma warning restore 618 async Task StartupTask(CancellationToken cancellationToken) { if (!await sqsQueue.ExistsAsync(cancellationToken).ConfigureAwait(false)) { await sqsQueue.CreateAsync(writeConfiguration, cancellationToken : cancellationToken).ConfigureAwait(false); } eventPublisher.QueueUrl = sqsQueue.Uri; } bus.AddStartupTask(StartupTask); bus.AddMessagePublisher <T>(eventPublisher); logger.LogInformation( "Created SQS publisher for message type '{MessageType}' on queue '{QueueName}'.", typeof(T), writeConfiguration.QueueName); }
public async Task SqsPolicyWithAWildcardIsApplied() { var queue = new SqsQueueByName(RegionEndpoint.EUWest1, QueueName, Client, 0, Substitute.For <ILoggerFactory>()); await Patiently.AssertThatAsync(() => queue.ExistsAsync(), TimeSpan.FromSeconds(60)); dynamic policyJson = JObject.Parse(queue.Policy); policyJson.Statement.Count.ShouldBe(1, $"Expecting 1 statement in Sqs policy but found {policyJson.Statement.Count}"); }
protected override void Given() { int retryCount = 1; _client = Substitute.For <IAmazonSQS>(); var response = GenerateResponseMessage(MessageTypeString, Guid.NewGuid()); _client.ReceiveMessageAsync( Arg.Any <ReceiveMessageRequest>(), Arg.Any <CancellationToken>()) .Returns( x => Task.FromResult(response), x => Task.FromResult(new ReceiveMessageResponse())); _client.GetQueueUrlAsync(Arg.Any <string>()) .Returns(x => { if (x.Arg <string>() == "some-queue-name") { return new GetQueueUrlResponse { QueueUrl = "https://testqueues.com/some-queue-name" } } ; throw new QueueDoesNotExistException("some-queue-name not found"); }); _client.GetQueueAttributesAsync(Arg.Any <GetQueueAttributesRequest>()) .Returns(new GetQueueAttributesResponse() { Attributes = new Dictionary <string, string> { { "QueueArn", "something:some-queue-name" } } }); var queue = new SqsQueueByName(RegionEndpoint.EUWest1, "some-queue-name", _client, retryCount, LoggerFactory); queue.ExistsAsync().Wait(); _queue = queue; Queues.Add(_queue); Handler.Handle(null) .ReturnsForAnyArgs(true).AndDoes(ci => Interlocked.Increment(ref _callCount)); }
public async Task QueueIsCreated() { async Task QueueIsCreatedInner() { var queue = new SqsQueueByName(RegionEndpoint.EUWest1, QueueName, Client, 0, Substitute.For <ILoggerFactory>()); await Patiently.AssertThatAsync( () => queue.ExistsAsync(), TimeSpan.FromSeconds(65)); } var task = QueueIsCreatedInner(); if (task == await Task.WhenAny(task, Task.Delay(TimeSpan.FromSeconds(70)))) // ToDo: Sorry about this, but SQS is a little slow to verify against. Can be better I'm sure? ;) { await task; } else { throw new TimeoutException(); } }
protected override void Given() { int retryCount = 1; _client = new FakeAmazonSqs(() => { return(new[] { GenerateResponseMessages(MessageTypeString, Guid.NewGuid()) } .Concat(new ReceiveMessageResponse().Infinite())); }); var queue = new SqsQueueByName(RegionEndpoint.EUWest1, "some-queue-name", _client, retryCount, LoggerFactory); queue.ExistsAsync().Wait(); _queue = queue; Queues.Add(_queue); }
public async Task Messages_Are_Throttled_But_Still_Delivered(int throttleMessageCount) { // Arrange ILoggerFactory loggerFactory = OutputHelper.ToLoggerFactory(); IAwsClientFactory clientFactory = CreateClientFactory(); int retryCountBeforeSendingToErrorQueue = 1; var client = clientFactory.GetSqsClient(Region); var queue = new SqsQueueByName( Region, UniqueName, client, retryCountBeforeSendingToErrorQueue, loggerFactory); if (!await queue.ExistsAsync(CancellationToken.None)) { await queue.CreateAsync(new SqsBasicConfiguration()); if (!IsSimulator) { // Wait for up to 60 secs for queue creation to be guaranteed completed by AWS using var cts = new CancellationTokenSource(TimeSpan.FromMinutes(1)); while (!cts.IsCancellationRequested) { await Task.Delay(TimeSpan.FromSeconds(2)); if (await queue.ExistsAsync(CancellationToken.None)) { break; } } } } Assert.True(await queue.ExistsAsync(CancellationToken.None), "The queue was not created."); OutputHelper.WriteLine($"{DateTime.Now} - Adding {throttleMessageCount} messages to the queue."); var entriesAdded = 0; // Add some messages do { var entries = new List <SendMessageBatchRequestEntry>(); for (int j = 0; j < 10; j++) { var batchEntry = new SendMessageBatchRequestEntry { MessageBody = $"{{\"Subject\":\"SimpleMessage\", \"Message\": {{ \"Content\": \"{entriesAdded}\"}}}}", Id = Guid.NewGuid().ToString() }; entries.Add(batchEntry); entriesAdded++; } await client.SendMessageBatchAsync(queue.Uri.AbsoluteUri, entries); }while (entriesAdded < throttleMessageCount); OutputHelper.WriteLine($"{DateTime.Now} - Done adding messages."); int count = 0; var handler = Substitute.For <IHandlerAsync <SimpleMessage> >(); handler.Handle(Arg.Any <SimpleMessage>()) .ReturnsForAnyArgs(true) .AndDoes((_) => Interlocked.Increment(ref count)); IServiceCollection services = GivenJustSaying(LogLevel.Warning) .ConfigureJustSaying((builder) => builder.WithLoopbackQueue <SimpleMessage>(UniqueName)) .AddSingleton(handler); TimeSpan timeToProcess = TimeSpan.Zero; // Act await WhenAsync( services, async (publisher, listener, cancellationToken) => { var stopwatch = Stopwatch.StartNew(); var delay = IsSimulator ? TimeSpan.FromMilliseconds(100) : TimeSpan.FromSeconds(5); await listener.StartAsync(cancellationToken); do { await Task.Delay(delay, cancellationToken); OutputHelper.WriteLine($"{DateTime.Now} - Handled {count} messages. Waiting for completion."); }while (count < throttleMessageCount && !cancellationToken.IsCancellationRequested); stopwatch.Stop(); timeToProcess = stopwatch.Elapsed; }); // Assert OutputHelper.WriteLine($"{DateTime.Now} - Handled {count:N0} messages."); OutputHelper.WriteLine($"{DateTime.Now} - Took {timeToProcess.TotalMilliseconds} ms"); OutputHelper.WriteLine($"{DateTime.Now} - Throughput {(float)count / timeToProcess.TotalMilliseconds * 1000} messages/second"); Assert.Equal(throttleMessageCount, count); }
// Use this to manually test the performance / throttling of getting messages out of the queue. public async Task HandlingManyMessages(int throttleMessageCount) { var locker = new object(); var awsQueueClient = CreateMeABus.DefaultClientFactory().GetSqsClient(RegionEndpoint.EUWest1); var q = new SqsQueueByName(RegionEndpoint.EUWest1, "throttle_test", awsQueueClient, 1, new LoggerFactory()); if (!await q.ExistsAsync()) { await q.CreateAsync(new SqsBasicConfiguration()); await Task.Delay(TimeSpan.FromMinutes(1)); // wait 60 secs for queue creation to be guaranteed completed by aws. :( } Assert.True(await q.ExistsAsync()); Console.WriteLine($"{DateTime.Now} - Adding {throttleMessageCount} messages to the queue."); var entriesAdded = 0; // Add some messages do { var entries = new List <SendMessageBatchRequestEntry>(); for (var j = 0; j < 10; j++) { var batchEntry = new SendMessageBatchRequestEntry { MessageBody = "{\"Subject\":\"GenericMessage\", \"Message\": \"" + entriesAdded.ToString() + "\"}", Id = Guid.NewGuid().ToString() }; entries.Add(batchEntry); entriesAdded++; } await awsQueueClient.SendMessageBatchAsync(new SendMessageBatchRequest { QueueUrl = q.Url, Entries = entries }); }while (entriesAdded < throttleMessageCount); Console.WriteLine($"{DateTime.Now} - Done adding messages."); var handleCount = 0; var serialisations = Substitute.For <IMessageSerialisationRegister>(); var monitor = Substitute.For <IMessageMonitor>(); var handler = Substitute.For <IHandlerAsync <GenericMessage> >(); handler.Handle(null).ReturnsForAnyArgs(true).AndDoes(x => { lock (locker) { handleCount++; } }); serialisations.DeserializeMessage(string.Empty).ReturnsForAnyArgs(new GenericMessage()); var listener = new SqsNotificationListener(q, serialisations, monitor, new LoggerFactory()); listener.AddMessageHandler(() => handler); var stopwatch = new Stopwatch(); stopwatch.Start(); listener.Listen(); var waitCount = 0; do { await Task.Delay(TimeSpan.FromSeconds(5)); Console.WriteLine($"{DateTime.Now} - Handled {handleCount} messages. Waiting for completion."); waitCount++; }while (handleCount < throttleMessageCount && waitCount < 100); listener.StopListening(); stopwatch.Stop(); Console.WriteLine($"{DateTime.Now} - Handled {handleCount} messages."); Console.WriteLine($"{DateTime.Now} - Took {stopwatch.ElapsedMilliseconds} ms"); Console.WriteLine( $"{DateTime.Now} - Throughput {(float) handleCount/stopwatch.ElapsedMilliseconds*1000} msg/sec"); Assert.Equal(throttleMessageCount, handleCount); }
public async Task IncorrectPartialQueueNameDoNotMatch() { var sqsQueueByName = new SqsQueueByName(RegionEndpoint.EUWest1, "some-queue", _client, RetryCount, _log); (await sqsQueueByName.ExistsAsync()).ShouldBeFalse(); }
public async Task IncorrectQueueNameDoNotMatch() { var sqsQueueByName = new SqsQueueByName(RegionEndpoint.EUWest1, "some-queue-name1", _client, RetryCount, _log); (await sqsQueueByName.ExistsAsync(CancellationToken.None)).ShouldBeFalse(); }
public void CorrectQueueNameShouldMatch() { var sqsQueueByName = new SqsQueueByName(RegionEndpoint.EUWest1, "some-queue-name", _client, RetryCount, _log); sqsQueueByName.ExistsAsync().GetAwaiter().GetResult().ShouldBeTrue(); }
public void IncorrectPartialQueueNameDoNotMatch() { var sqsQueueByName = new SqsQueueByName(RegionEndpoint.EUWest1, "some-queue", _client, RetryCount, _log); sqsQueueByName.ExistsAsync().GetAwaiter().GetResult().ShouldBeFalse(); }
private async Task AssertThatQueueDoesNotExist(string name) { var sqsQueueByName = new SqsQueueByName(Region, name, _client, 1, LoggerFactory); (await sqsQueueByName.ExistsAsync()).ShouldBeFalse($"Expecting queue '{name}' to not exist but it does."); }
public async Task HandlingManyMessages(int throttleMessageCount) { // Arrange var fixture = new JustSayingFixture(OutputHelper); var client = fixture.CreateSqsClient(); var queue = new SqsQueueByName(fixture.Region, fixture.UniqueName, client, 1, fixture.LoggerFactory); if (!await queue.ExistsAsync()) { await queue.CreateAsync(new SqsBasicConfiguration()); if (!fixture.IsSimulator) { // Wait for up to 60 secs for queue creation to be guaranteed completed by AWS using (var cts = new CancellationTokenSource(TimeSpan.FromMinutes(1))) { while (!cts.IsCancellationRequested) { await Task.Delay(TimeSpan.FromSeconds(2)); if (await queue.ExistsAsync()) { break; } } } } } Assert.True(await queue.ExistsAsync(), "The queue was not created."); OutputHelper.WriteLine($"{DateTime.Now} - Adding {throttleMessageCount} messages to the queue."); var entriesAdded = 0; // Add some messages do { var entries = new List <SendMessageBatchRequestEntry>(); for (var j = 0; j < 10; j++) { var batchEntry = new SendMessageBatchRequestEntry { MessageBody = $"{{\"Subject\":\"GenericMessage\", \"Message\": \"{entriesAdded}\"}}", Id = Guid.NewGuid().ToString() }; entries.Add(batchEntry); entriesAdded++; } await client.SendMessageBatchAsync(queue.Url, entries); }while (entriesAdded < throttleMessageCount); OutputHelper.WriteLine($"{DateTime.Now} - Done adding messages."); var handleCount = 0; var serialisations = Substitute.For <IMessageSerialisationRegister>(); var monitor = Substitute.For <IMessageMonitor>(); var handler = Substitute.For <IHandlerAsync <SimpleMessage> >(); handler.Handle(null).ReturnsForAnyArgs(true).AndDoes(_ => Interlocked.Increment(ref handleCount)); serialisations.DeserializeMessage(string.Empty).ReturnsForAnyArgs(new SimpleMessage()); var listener = new SqsNotificationListener(queue, serialisations, monitor, fixture.LoggerFactory); listener.AddMessageHandler(() => handler); // Act var stopwatch = Stopwatch.StartNew(); listener.Listen(); using (var cts = new CancellationTokenSource(TimeSpan.FromMinutes(5))) { do { if (!fixture.IsSimulator) { await Task.Delay(TimeSpan.FromSeconds(5)); } OutputHelper.WriteLine($"{DateTime.Now} - Handled {handleCount} messages. Waiting for completion."); }while (handleCount < throttleMessageCount && !cts.IsCancellationRequested); } listener.StopListening(); stopwatch.Stop(); OutputHelper.WriteLine($"{DateTime.Now} - Handled {handleCount:N0} messages."); OutputHelper.WriteLine($"{DateTime.Now} - Took {stopwatch.ElapsedMilliseconds} ms"); OutputHelper.WriteLine($"{DateTime.Now} - Throughput {(float)handleCount / stopwatch.ElapsedMilliseconds * 1000} messages/second"); // Assert Assert.Equal(throttleMessageCount, handleCount); }
public async Task CorrectQueueNameShouldMatch() { var sqsQueueByName = new SqsQueueByName(RegionEndpoint.EUWest1, "some-queue-name", _client, RetryCount, _log); (await sqsQueueByName.ExistsAsync()).ShouldBeTrue(); }
private void AssertThatQueueDoesNotExist(string name) { var sqsQueueByName = new SqsQueueByName(RegionEndpoint.EUWest1, name, _client, 1, new LoggerFactory()); sqsQueueByName.ExistsAsync().GetAwaiter().GetResult().ShouldBeFalse($"Expecting queue '{name}' to not exist but it does."); }