public async Task Sqs_Policy_Is_Applied_With_Wildcard()
{
// Arrange
var handler = new ExactlyOnceHandlerWithTimeout();
var services = GivenJustSaying()
.ConfigureJustSaying((builder) => builder.WithLoopbackTopic <TopicA>(UniqueName))
.ConfigureJustSaying((builder) => builder.WithLoopbackTopic <TopicB>(UniqueName))
.AddJustSayingHandler <TopicA, HandlerA>()
.AddJustSayingHandler <TopicB, HandlerB>();
await WhenAsync(
services,
async (publisher, listener, serviceProvider, cancellationToken) =>
{
listener.Start(cancellationToken);
var clientFactory = serviceProvider.GetRequiredService <MessagingBusBuilder>().BuildClientFactory();
var loggerFactory = serviceProvider.GetRequiredService <ILoggerFactory>();
var client = clientFactory.GetSqsClient(Region);
var queue = new SqsQueueByName(Region, UniqueName, client, 0, loggerFactory);
await Patiently.AssertThatAsync(() => queue.ExistsAsync(), 60.Seconds());
dynamic policyJson = JObject.Parse(queue.Policy);
policyJson.Statement.Count.ShouldBe(1, $"Expecting 1 statement in Sqs policy but found {policyJson.Statement.Count}.");
});
}
private static async Task EnsureQueueExistsAsync(SqsQueueByName queue)
{
if (!await queue.ExistsAsync().ConfigureAwait(false))
{
throw new InvalidOperationException($"{queue.QueueName} does not exist.");
}
}
public async Task QueueIsCreated()
{
async Task QueueIsCreatedInner()
{
var queue = new SqsQueueByName(
TestFixture.Region,
QueueName,
Client,
0,
TestFixture.LoggerFactory);
await Patiently.AssertThatAsync(
() => queue.ExistsAsync(), TimeSpan.FromSeconds(65));
}
var task = QueueIsCreatedInner();
if (task == await Task.WhenAny(task, Task.Delay(TimeSpan.FromSeconds(70))))
{
await task;
}
else
{
throw new TimeoutException();
}
}
/// <inheritdoc />
void IPublicationBuilder <T> .Configure(
JustSayingBus bus,
IAwsClientFactoryProxy proxy,
ILoggerFactory loggerFactory)
{
var logger = loggerFactory.CreateLogger <QueuePublicationBuilder <T> >();
logger.LogInformation("Adding SQS publisher for message type '{MessageType}'.",
typeof(T));
var config = bus.Config;
var region = config.Region ?? throw new InvalidOperationException($"Config cannot have a blank entry for the {nameof(config.Region)} property.");
var writeConfiguration = new SqsWriteConfiguration();
ConfigureWrites?.Invoke(writeConfiguration);
writeConfiguration.ApplyQueueNamingConvention <T>(config.QueueNamingConvention);
bus.SerializationRegister.AddSerializer <T>();
var regionEndpoint = RegionEndpoint.GetBySystemName(region);
var sqsClient = proxy.GetAwsClientFactory().GetSqsClient(regionEndpoint);
var eventPublisher = new SqsMessagePublisher(
sqsClient,
bus.SerializationRegister,
loggerFactory)
{
MessageResponseLogger = config.MessageResponseLogger
};
#pragma warning disable 618
var sqsQueue = new SqsQueueByName(
regionEndpoint,
writeConfiguration.QueueName,
sqsClient,
writeConfiguration.RetryCountBeforeSendingToErrorQueue,
loggerFactory);
#pragma warning restore 618
async Task StartupTask(CancellationToken cancellationToken)
{
if (!await sqsQueue.ExistsAsync(cancellationToken).ConfigureAwait(false))
{
await sqsQueue.CreateAsync(writeConfiguration, cancellationToken : cancellationToken).ConfigureAwait(false);
}
eventPublisher.QueueUrl = sqsQueue.Uri;
}
bus.AddStartupTask(StartupTask);
bus.AddMessagePublisher <T>(eventPublisher);
logger.LogInformation(
"Created SQS publisher for message type '{MessageType}' on queue '{QueueName}'.",
typeof(T),
writeConfiguration.QueueName);
}
public async Task SqsPolicyWithAWildcardIsApplied()
{
var queue = new SqsQueueByName(RegionEndpoint.EUWest1, QueueName, Client, 0, Substitute.For <ILoggerFactory>());
await Patiently.AssertThatAsync(() => queue.ExistsAsync(), TimeSpan.FromSeconds(60));
dynamic policyJson = JObject.Parse(queue.Policy);
policyJson.Statement.Count.ShouldBe(1, $"Expecting 1 statement in Sqs policy but found {policyJson.Statement.Count}");
}
protected override void Given()
{
int retryCount = 1;
_client = Substitute.For <IAmazonSQS>();
var response = GenerateResponseMessage(MessageTypeString, Guid.NewGuid());
_client.ReceiveMessageAsync(
Arg.Any <ReceiveMessageRequest>(),
Arg.Any <CancellationToken>())
.Returns(
x => Task.FromResult(response),
x => Task.FromResult(new ReceiveMessageResponse()));
_client.GetQueueUrlAsync(Arg.Any <string>())
.Returns(x =>
{
if (x.Arg <string>() == "some-queue-name")
{
return new GetQueueUrlResponse
{
QueueUrl = "https://testqueues.com/some-queue-name"
}
}
;
throw new QueueDoesNotExistException("some-queue-name not found");
});
_client.GetQueueAttributesAsync(Arg.Any <GetQueueAttributesRequest>())
.Returns(new GetQueueAttributesResponse()
{
Attributes = new Dictionary <string, string> {
{ "QueueArn", "something:some-queue-name" }
}
});
var queue = new SqsQueueByName(RegionEndpoint.EUWest1, "some-queue-name", _client, retryCount, LoggerFactory);
queue.ExistsAsync().Wait();
_queue = queue;
Queues.Add(_queue);
Handler.Handle(null)
.ReturnsForAnyArgs(true).AndDoes(ci => Interlocked.Increment(ref _callCount));
}
public async Task QueueIsCreated()
{
async Task QueueIsCreatedInner()
{
var queue = new SqsQueueByName(RegionEndpoint.EUWest1,
QueueName, Client, 0, Substitute.For <ILoggerFactory>());
await Patiently.AssertThatAsync(
() => queue.ExistsAsync(), TimeSpan.FromSeconds(65));
}
var task = QueueIsCreatedInner();
if (task == await Task.WhenAny(task, Task.Delay(TimeSpan.FromSeconds(70)))) // ToDo: Sorry about this, but SQS is a little slow to verify against. Can be better I'm sure? ;)
{
await task;
}
else
{
throw new TimeoutException();
}
}
protected override void Given()
{
int retryCount = 1;
_client = new FakeAmazonSqs(() =>
{
return(new[] { GenerateResponseMessages(MessageTypeString, Guid.NewGuid()) }
.Concat(new ReceiveMessageResponse().Infinite()));
});
var queue = new SqsQueueByName(RegionEndpoint.EUWest1,
"some-queue-name",
_client,
retryCount,
LoggerFactory);
queue.ExistsAsync().Wait();
_queue = queue;
Queues.Add(_queue);
}
public async Task Messages_Are_Throttled_But_Still_Delivered(int throttleMessageCount)
{
// Arrange
ILoggerFactory loggerFactory = OutputHelper.ToLoggerFactory();
IAwsClientFactory clientFactory = CreateClientFactory();
int retryCountBeforeSendingToErrorQueue = 1;
var client = clientFactory.GetSqsClient(Region);
var queue = new SqsQueueByName(
Region,
UniqueName,
client,
retryCountBeforeSendingToErrorQueue,
loggerFactory);
if (!await queue.ExistsAsync(CancellationToken.None))
{
await queue.CreateAsync(new SqsBasicConfiguration());
if (!IsSimulator)
{
// Wait for up to 60 secs for queue creation to be guaranteed completed by AWS
using var cts = new CancellationTokenSource(TimeSpan.FromMinutes(1));
while (!cts.IsCancellationRequested)
{
await Task.Delay(TimeSpan.FromSeconds(2));
if (await queue.ExistsAsync(CancellationToken.None))
{
break;
}
}
}
}
Assert.True(await queue.ExistsAsync(CancellationToken.None), "The queue was not created.");
OutputHelper.WriteLine($"{DateTime.Now} - Adding {throttleMessageCount} messages to the queue.");
var entriesAdded = 0;
// Add some messages
do
{
var entries = new List <SendMessageBatchRequestEntry>();
for (int j = 0; j < 10; j++)
{
var batchEntry = new SendMessageBatchRequestEntry
{
MessageBody = $"{{\"Subject\":\"SimpleMessage\", \"Message\": {{ \"Content\": \"{entriesAdded}\"}}}}",
Id = Guid.NewGuid().ToString()
};
entries.Add(batchEntry);
entriesAdded++;
}
await client.SendMessageBatchAsync(queue.Uri.AbsoluteUri, entries);
}while (entriesAdded < throttleMessageCount);
OutputHelper.WriteLine($"{DateTime.Now} - Done adding messages.");
int count = 0;
var handler = Substitute.For <IHandlerAsync <SimpleMessage> >();
handler.Handle(Arg.Any <SimpleMessage>())
.ReturnsForAnyArgs(true)
.AndDoes((_) => Interlocked.Increment(ref count));
IServiceCollection services = GivenJustSaying(LogLevel.Warning)
.ConfigureJustSaying((builder) => builder.WithLoopbackQueue <SimpleMessage>(UniqueName))
.AddSingleton(handler);
TimeSpan timeToProcess = TimeSpan.Zero;
// Act
await WhenAsync(
services,
async (publisher, listener, cancellationToken) =>
{
var stopwatch = Stopwatch.StartNew();
var delay = IsSimulator ? TimeSpan.FromMilliseconds(100) : TimeSpan.FromSeconds(5);
await listener.StartAsync(cancellationToken);
do
{
await Task.Delay(delay, cancellationToken);
OutputHelper.WriteLine($"{DateTime.Now} - Handled {count} messages. Waiting for completion.");
}while (count < throttleMessageCount && !cancellationToken.IsCancellationRequested);
stopwatch.Stop();
timeToProcess = stopwatch.Elapsed;
});
// Assert
OutputHelper.WriteLine($"{DateTime.Now} - Handled {count:N0} messages.");
OutputHelper.WriteLine($"{DateTime.Now} - Took {timeToProcess.TotalMilliseconds} ms");
OutputHelper.WriteLine($"{DateTime.Now} - Throughput {(float)count / timeToProcess.TotalMilliseconds * 1000} messages/second");
Assert.Equal(throttleMessageCount, count);
}
// Use this to manually test the performance / throttling of getting messages out of the queue.
public async Task HandlingManyMessages(int throttleMessageCount)
{
var locker = new object();
var awsQueueClient = CreateMeABus.DefaultClientFactory().GetSqsClient(RegionEndpoint.EUWest1);
var q = new SqsQueueByName(RegionEndpoint.EUWest1, "throttle_test", awsQueueClient, 1, new LoggerFactory());
if (!await q.ExistsAsync())
{
await q.CreateAsync(new SqsBasicConfiguration());
await Task.Delay(TimeSpan.FromMinutes(1)); // wait 60 secs for queue creation to be guaranteed completed by aws. :(
}
Assert.True(await q.ExistsAsync());
Console.WriteLine($"{DateTime.Now} - Adding {throttleMessageCount} messages to the queue.");
var entriesAdded = 0;
// Add some messages
do
{
var entries = new List <SendMessageBatchRequestEntry>();
for (var j = 0; j < 10; j++)
{
var batchEntry = new SendMessageBatchRequestEntry
{
MessageBody = "{\"Subject\":\"GenericMessage\", \"Message\": \"" + entriesAdded.ToString() + "\"}",
Id = Guid.NewGuid().ToString()
};
entries.Add(batchEntry);
entriesAdded++;
}
await awsQueueClient.SendMessageBatchAsync(new SendMessageBatchRequest { QueueUrl = q.Url, Entries = entries });
}while (entriesAdded < throttleMessageCount);
Console.WriteLine($"{DateTime.Now} - Done adding messages.");
var handleCount = 0;
var serialisations = Substitute.For <IMessageSerialisationRegister>();
var monitor = Substitute.For <IMessageMonitor>();
var handler = Substitute.For <IHandlerAsync <GenericMessage> >();
handler.Handle(null).ReturnsForAnyArgs(true).AndDoes(x => { lock (locker) { handleCount++; } });
serialisations.DeserializeMessage(string.Empty).ReturnsForAnyArgs(new GenericMessage());
var listener = new SqsNotificationListener(q, serialisations, monitor, new LoggerFactory());
listener.AddMessageHandler(() => handler);
var stopwatch = new Stopwatch();
stopwatch.Start();
listener.Listen();
var waitCount = 0;
do
{
await Task.Delay(TimeSpan.FromSeconds(5));
Console.WriteLine($"{DateTime.Now} - Handled {handleCount} messages. Waiting for completion.");
waitCount++;
}while (handleCount < throttleMessageCount && waitCount < 100);
listener.StopListening();
stopwatch.Stop();
Console.WriteLine($"{DateTime.Now} - Handled {handleCount} messages.");
Console.WriteLine($"{DateTime.Now} - Took {stopwatch.ElapsedMilliseconds} ms");
Console.WriteLine(
$"{DateTime.Now} - Throughput {(float) handleCount/stopwatch.ElapsedMilliseconds*1000} msg/sec");
Assert.Equal(throttleMessageCount, handleCount);
}
public async Task IncorrectPartialQueueNameDoNotMatch()
{
var sqsQueueByName = new SqsQueueByName(RegionEndpoint.EUWest1, "some-queue", _client, RetryCount, _log);
(await sqsQueueByName.ExistsAsync()).ShouldBeFalse();
}
public async Task IncorrectQueueNameDoNotMatch()
{
var sqsQueueByName = new SqsQueueByName(RegionEndpoint.EUWest1, "some-queue-name1", _client, RetryCount, _log);
(await sqsQueueByName.ExistsAsync(CancellationToken.None)).ShouldBeFalse();
}
public void CorrectQueueNameShouldMatch()
{
var sqsQueueByName = new SqsQueueByName(RegionEndpoint.EUWest1, "some-queue-name", _client, RetryCount, _log);
sqsQueueByName.ExistsAsync().GetAwaiter().GetResult().ShouldBeTrue();
}
public void IncorrectPartialQueueNameDoNotMatch()
{
var sqsQueueByName = new SqsQueueByName(RegionEndpoint.EUWest1, "some-queue", _client, RetryCount, _log);
sqsQueueByName.ExistsAsync().GetAwaiter().GetResult().ShouldBeFalse();
}
private async Task AssertThatQueueDoesNotExist(string name)
{
var sqsQueueByName = new SqsQueueByName(Region, name, _client, 1, LoggerFactory);
(await sqsQueueByName.ExistsAsync()).ShouldBeFalse($"Expecting queue '{name}' to not exist but it does.");
}
public async Task HandlingManyMessages(int throttleMessageCount)
{
// Arrange
var fixture = new JustSayingFixture(OutputHelper);
var client = fixture.CreateSqsClient();
var queue = new SqsQueueByName(fixture.Region, fixture.UniqueName, client, 1, fixture.LoggerFactory);
if (!await queue.ExistsAsync())
{
await queue.CreateAsync(new SqsBasicConfiguration());
if (!fixture.IsSimulator)
{
// Wait for up to 60 secs for queue creation to be guaranteed completed by AWS
using (var cts = new CancellationTokenSource(TimeSpan.FromMinutes(1)))
{
while (!cts.IsCancellationRequested)
{
await Task.Delay(TimeSpan.FromSeconds(2));
if (await queue.ExistsAsync())
{
break;
}
}
}
}
}
Assert.True(await queue.ExistsAsync(), "The queue was not created.");
OutputHelper.WriteLine($"{DateTime.Now} - Adding {throttleMessageCount} messages to the queue.");
var entriesAdded = 0;
// Add some messages
do
{
var entries = new List <SendMessageBatchRequestEntry>();
for (var j = 0; j < 10; j++)
{
var batchEntry = new SendMessageBatchRequestEntry
{
MessageBody = $"{{\"Subject\":\"GenericMessage\", \"Message\": \"{entriesAdded}\"}}",
Id = Guid.NewGuid().ToString()
};
entries.Add(batchEntry);
entriesAdded++;
}
await client.SendMessageBatchAsync(queue.Url, entries);
}while (entriesAdded < throttleMessageCount);
OutputHelper.WriteLine($"{DateTime.Now} - Done adding messages.");
var handleCount = 0;
var serialisations = Substitute.For <IMessageSerialisationRegister>();
var monitor = Substitute.For <IMessageMonitor>();
var handler = Substitute.For <IHandlerAsync <SimpleMessage> >();
handler.Handle(null).ReturnsForAnyArgs(true).AndDoes(_ => Interlocked.Increment(ref handleCount));
serialisations.DeserializeMessage(string.Empty).ReturnsForAnyArgs(new SimpleMessage());
var listener = new SqsNotificationListener(queue, serialisations, monitor, fixture.LoggerFactory);
listener.AddMessageHandler(() => handler);
// Act
var stopwatch = Stopwatch.StartNew();
listener.Listen();
using (var cts = new CancellationTokenSource(TimeSpan.FromMinutes(5)))
{
do
{
if (!fixture.IsSimulator)
{
await Task.Delay(TimeSpan.FromSeconds(5));
}
OutputHelper.WriteLine($"{DateTime.Now} - Handled {handleCount} messages. Waiting for completion.");
}while (handleCount < throttleMessageCount && !cts.IsCancellationRequested);
}
listener.StopListening();
stopwatch.Stop();
OutputHelper.WriteLine($"{DateTime.Now} - Handled {handleCount:N0} messages.");
OutputHelper.WriteLine($"{DateTime.Now} - Took {stopwatch.ElapsedMilliseconds} ms");
OutputHelper.WriteLine($"{DateTime.Now} - Throughput {(float)handleCount / stopwatch.ElapsedMilliseconds * 1000} messages/second");
// Assert
Assert.Equal(throttleMessageCount, handleCount);
}
public async Task CorrectQueueNameShouldMatch()
{
var sqsQueueByName = new SqsQueueByName(RegionEndpoint.EUWest1, "some-queue-name", _client, RetryCount, _log);
(await sqsQueueByName.ExistsAsync()).ShouldBeTrue();
}
private void AssertThatQueueDoesNotExist(string name)
{
var sqsQueueByName = new SqsQueueByName(RegionEndpoint.EUWest1, name, _client, 1, new LoggerFactory());
sqsQueueByName.ExistsAsync().GetAwaiter().GetResult().ShouldBeFalse($"Expecting queue '{name}' to not exist but it does.");
}
