public async Task Sqs_Policy_Is_Applied_With_Wildcard()
{
// Arrange
var handler = new ExactlyOnceHandlerWithTimeout();
var services = GivenJustSaying()
.ConfigureJustSaying((builder) => builder.WithLoopbackTopic <TopicA>(UniqueName))
.ConfigureJustSaying((builder) => builder.WithLoopbackTopic <TopicB>(UniqueName))
.AddJustSayingHandler <TopicA, HandlerA>()
.AddJustSayingHandler <TopicB, HandlerB>();
await WhenAsync(
services,
async (publisher, listener, serviceProvider, cancellationToken) =>
{
listener.Start(cancellationToken);
var clientFactory = serviceProvider.GetRequiredService <MessagingBusBuilder>().BuildClientFactory();
var loggerFactory = serviceProvider.GetRequiredService <ILoggerFactory>();
var client = clientFactory.GetSqsClient(Region);
var queue = new SqsQueueByName(Region, UniqueName, client, 0, loggerFactory);
await Patiently.AssertThatAsync(() => queue.ExistsAsync(), 60.Seconds());
dynamic policyJson = JObject.Parse(queue.Policy);
policyJson.Statement.Count.ShouldBe(1, $"Expecting 1 statement in Sqs policy but found {policyJson.Statement.Count}.");
});
}
public async Task Then_The_Error_Queue_Is_Not_Created()
{
// Arrange
ILoggerFactory loggerFactory = OutputHelper.ToLoggerFactory();
IAwsClientFactory clientFactory = CreateClientFactory();
var client = clientFactory.GetSqsClient(Region);
var queue = new SqsQueueByName(
Region,
UniqueName,
client,
1,
loggerFactory);
// Act
await queue.CreateAsync(new SqsBasicConfiguration()
{
ErrorQueueOptOut = true
});
// Assert
await Patiently.AssertThatAsync(
OutputHelper, async() => !await queue.ErrorQueue.ExistsAsync(CancellationToken.None));
}
public SqsQueueByName VerifyOrCreateQueue(string region, IMessageSerialisationRegister serialisationRegister, SqsConfiguration queueConfig)
{
var sqsclient = AWSClientFactory.CreateAmazonSQSClient(RegionEndpoint.GetBySystemName(region));
var snsclient = AWSClientFactory.CreateAmazonSimpleNotificationServiceClient(RegionEndpoint.GetBySystemName(region));
var queue = new SqsQueueByName(queueConfig.QueueName, sqsclient, queueConfig.RetryCountBeforeSendingToErrorQueue);
var eventTopic = new SnsTopicByName(queueConfig.PublishEndpoint, snsclient, serialisationRegister);
if (!queue.Exists())
queue.Create(queueConfig.MessageRetentionSeconds, 0, queueConfig.VisibilityTimeoutSeconds, queueConfig.ErrorQueueOptOut, queueConfig.RetryCountBeforeSendingToErrorQueue);
//Create an error queue for existing queues if they don't already have one
if(queue.ErrorQueue != null && !queue.ErrorQueue.Exists())
queue.ErrorQueue.Create(JustSayingConstants.MAXIMUM_RETENTION_PERIOD, JustSayingConstants.DEFAULT_CREATE_REATTEMPT, JustSayingConstants.DEFAULT_VISIBILITY_TIMEOUT, errorQueueOptOut: true);
queue.UpdateRedrivePolicy(new RedrivePolicy(queueConfig.RetryCountBeforeSendingToErrorQueue, queue.ErrorQueue.Arn));
if (!eventTopic.Exists())
eventTopic.Create();
if (!eventTopic.IsSubscribed(queue))
eventTopic.Subscribe(queue);
if (!queue.HasPermission(eventTopic))
queue.AddPermission(eventTopic);
return queue;
}
public SqsQueueByName EnsureQueueExists(string region, SqsReadConfiguration queueConfig)
{
var sqsclient = AWSClientFactory.CreateAmazonSQSClient(RegionEndpoint.GetBySystemName(region));
var queue = new SqsQueueByName(queueConfig.QueueName, sqsclient, queueConfig.RetryCountBeforeSendingToErrorQueue);
queue.EnsureQueueAndErrorQueueExistAndAllAttributesAreUpdated(queueConfig);
return queue;
}
public async Task Can_Update_Redrive_Policy()
{
// Arrange
int maximumReceives = 42;
ILoggerFactory loggerFactory = OutputHelper.ToLoggerFactory();
IAwsClientFactory clientFactory = CreateClientFactory();
var client = clientFactory.GetSqsClient(Region);
var queue = new SqsQueueByName(
Region,
UniqueName,
client,
1,
loggerFactory);
await queue.CreateAsync(new SqsBasicConfiguration());
// Act
await queue.UpdateRedrivePolicyAsync(
new RedrivePolicy(maximumReceives, queue.ErrorQueue.Arn));
// Assert
queue.RedrivePolicy.ShouldNotBeNull();
queue.RedrivePolicy.MaximumReceives.ShouldBe(maximumReceives);
}
public async Task QueueIsCreated()
{
async Task QueueIsCreatedInner()
{
var queue = new SqsQueueByName(
TestFixture.Region,
QueueName,
Client,
0,
TestFixture.LoggerFactory);
await Patiently.AssertThatAsync(
() => queue.ExistsAsync(), TimeSpan.FromSeconds(65));
}
var task = QueueIsCreatedInner();
if (task == await Task.WhenAny(task, Task.Delay(TimeSpan.FromSeconds(70))))
{
await task;
}
else
{
throw new TimeoutException();
}
}
public async Task Can_Update_Retention_Period()
{
// Arrange
var oldRetentionPeriod = TimeSpan.FromSeconds(600);
var newRetentionPeriod = TimeSpan.FromSeconds(700);
ILoggerFactory loggerFactory = OutputHelper.ToLoggerFactory();
IAwsClientFactory clientFactory = CreateClientFactory();
var client = clientFactory.GetSqsClient(Region);
var queue = new SqsQueueByName(
Region,
UniqueName,
client,
1,
loggerFactory);
await queue.CreateAsync(
new SqsBasicConfiguration { MessageRetention = oldRetentionPeriod });
// Act
await queue.UpdateQueueAttributeAsync(
new SqsBasicConfiguration { MessageRetention = newRetentionPeriod }, CancellationToken.None);
// Assert
queue.MessageRetentionPeriod.ShouldBe(newRetentionPeriod);
}
public bool Execute()
{
Console.WriteLine("Moving {0} messages from {1} to {2}", Count, SourceQueueName, DestinationQueueName);
var config = new AmazonSQSConfig();
var client = new DefaultAwsClientFactory().GetSqsClient(config.RegionEndpoint);
var sourceQueue = new SqsQueueByName(config.RegionEndpoint, SourceQueueName, client, JustSayingConstants.DEFAULT_HANDLER_RETRY_COUNT);
var destinationQueue = new SqsQueueByName(config.RegionEndpoint, DestinationQueueName, client, JustSayingConstants.DEFAULT_HANDLER_RETRY_COUNT);
var messages = PopMessagesFromSourceQueue(sourceQueue);
var receiptHandles = messages.ToDictionary(m => m.MessageId, m => m.ReceiptHandle);
var sendResponse = destinationQueue.Client.SendMessageBatch(new SendMessageBatchRequest
{
QueueUrl = destinationQueue.Url,
Entries = messages.Select(x => new SendMessageBatchRequestEntry {
Id = x.MessageId, MessageBody = x.Body
}).ToList()
});
var deleteResponse = sourceQueue.Client.DeleteMessageBatch(new DeleteMessageBatchRequest
{
QueueUrl = sourceQueue.Url,
Entries = sendResponse.Successful.Select(x => new DeleteMessageBatchRequestEntry
{
Id = x.Id,
ReceiptHandle = receiptHandles[x.Id]
}).ToList()
});
Console.WriteLine("Moved {0} messages from {1} to {2}", sendResponse.Successful.Count, SourceQueueName, DestinationQueueName);
return(true);
}
private static async Task EnsureQueueExistsAsync(SqsQueueByName queue)
{
if (!await queue.ExistsAsync().ConfigureAwait(false))
{
throw new InvalidOperationException($"{queue.QueueName} does not exist.");
}
}
private void EnsureQueueExists(SqsQueueByName queue)
{
if (!queue.Exists())
{
throw new InvalidOperationException($"{queue.QueueName} does not exist.");
}
}
public async Task Can_Remove_Encryption()
{
// Arrange
ILoggerFactory loggerFactory = OutputHelper.ToLoggerFactory();
IAwsClientFactory clientFactory = CreateClientFactory();
var client = clientFactory.GetSqsClient(Region);
var queue = new SqsQueueByName(
Region,
UniqueName,
client,
1,
loggerFactory);
await queue.CreateAsync(
new SqsBasicConfiguration { ServerSideEncryption = new ServerSideEncryption() });
// Act
await queue.UpdateQueueAttributeAsync(
new SqsBasicConfiguration { ServerSideEncryption = null });
// Assert
queue.ServerSideEncryption.ShouldBeNull();
}
public async Task Can_Update_Delivery_Delay()
{
// Arrange
var oldDeliveryDelay = TimeSpan.FromMinutes(2);
var newDeliveryDelay = TimeSpan.FromMinutes(5);
ILoggerFactory loggerFactory = OutputHelper.ToLoggerFactory();
IAwsClientFactory clientFactory = CreateClientFactory();
var client = clientFactory.GetSqsClient(Region);
var queue = new SqsQueueByName(
Region,
UniqueName,
client,
1,
loggerFactory);
await queue.CreateAsync(
new SqsBasicConfiguration()
{
DeliveryDelay = oldDeliveryDelay
});
// Act
await queue.UpdateQueueAttributeAsync(
new SqsBasicConfiguration()
{
DeliveryDelay = newDeliveryDelay
});
// Assert
queue.DeliveryDelay.ShouldBe(newDeliveryDelay);
}
public bool Execute()
{
Console.WriteLine($"Moving {Count} messages from {SourceQueueName} to {DestinationQueueName} in {Region}.");
var config = new AmazonSQSConfig { RegionEndpoint = RegionEndpoint.GetBySystemName(Region) };
var client = new DefaultAwsClientFactory().GetSqsClient(config.RegionEndpoint);
var sourceQueue = new SqsQueueByName(config.RegionEndpoint, SourceQueueName, client, JustSayingConstants.DEFAULT_HANDLER_RETRY_COUNT);
var destinationQueue = new SqsQueueByName(config.RegionEndpoint, DestinationQueueName, client, JustSayingConstants.DEFAULT_HANDLER_RETRY_COUNT);
EnsureQueueExists(sourceQueue);
EnsureQueueExists(destinationQueue);
var messages = PopMessagesFromSourceQueue(sourceQueue);
var receiptHandles = messages.ToDictionary(m => m.MessageId, m => m.ReceiptHandle);
var sendResponse = destinationQueue.Client.SendMessageBatch(new SendMessageBatchRequest
{
QueueUrl = destinationQueue.Url,
Entries = messages.Select(x => new SendMessageBatchRequestEntry { Id = x.MessageId, MessageBody = x.Body }).ToList()
});
var deleteResponse = sourceQueue.Client.DeleteMessageBatch(new DeleteMessageBatchRequest
{
QueueUrl = sourceQueue.Url,
Entries = sendResponse.Successful.Select(x => new DeleteMessageBatchRequestEntry
{
Id = x.Id,
ReceiptHandle = receiptHandles[x.Id]
}).ToList()
});
Console.WriteLine($"Moved {sendResponse.Successful.Count} messages from {SourceQueueName} to {DestinationQueueName} in {Region}.");
return true;
}
public SqsQueueByName VerifyOrCreateQueue(string region, IMessageSerialisationRegister serialisationRegister, SqsConfiguration queueConfig)
{
var sqsclient = AWSClientFactory.CreateAmazonSQSClient(RegionEndpoint.GetBySystemName(region));
var snsclient = AWSClientFactory.CreateAmazonSimpleNotificationServiceClient(RegionEndpoint.GetBySystemName(region));
var queue = new SqsQueueByName(queueConfig.QueueName, sqsclient, queueConfig.RetryCountBeforeSendingToErrorQueue);
var eventTopic = new SnsTopicByName(queueConfig.PublishEndpoint, snsclient, serialisationRegister);
queue.EnsureQueueAndErrorQueueExistAndAllAttributesAreUpdated(queueConfig);
if (!eventTopic.Exists())
{
eventTopic.Create();
}
if (!eventTopic.IsSubscribed(queue))
{
eventTopic.Subscribe(queue);
}
if (!queue.HasPermission(eventTopic))
{
queue.AddPermission(eventTopic);
}
return(queue);
}
/// <inheritdoc />
void IPublicationBuilder <T> .Configure(
JustSayingBus bus,
IAwsClientFactoryProxy proxy,
ILoggerFactory loggerFactory)
{
var logger = loggerFactory.CreateLogger <QueuePublicationBuilder <T> >();
logger.LogInformation("Adding SQS publisher for message type '{MessageType}'.",
typeof(T));
var config = bus.Config;
var region = config.Region ?? throw new InvalidOperationException($"Config cannot have a blank entry for the {nameof(config.Region)} property.");
var writeConfiguration = new SqsWriteConfiguration();
ConfigureWrites?.Invoke(writeConfiguration);
writeConfiguration.ApplyQueueNamingConvention <T>(config.QueueNamingConvention);
bus.SerializationRegister.AddSerializer <T>();
var regionEndpoint = RegionEndpoint.GetBySystemName(region);
var sqsClient = proxy.GetAwsClientFactory().GetSqsClient(regionEndpoint);
var eventPublisher = new SqsMessagePublisher(
sqsClient,
bus.SerializationRegister,
loggerFactory)
{
MessageResponseLogger = config.MessageResponseLogger
};
#pragma warning disable 618
var sqsQueue = new SqsQueueByName(
regionEndpoint,
writeConfiguration.QueueName,
sqsClient,
writeConfiguration.RetryCountBeforeSendingToErrorQueue,
loggerFactory);
#pragma warning restore 618
async Task StartupTask(CancellationToken cancellationToken)
{
if (!await sqsQueue.ExistsAsync(cancellationToken).ConfigureAwait(false))
{
await sqsQueue.CreateAsync(writeConfiguration, cancellationToken : cancellationToken).ConfigureAwait(false);
}
eventPublisher.QueueUrl = sqsQueue.Uri;
}
bus.AddStartupTask(StartupTask);
bus.AddMessagePublisher <T>(eventPublisher);
logger.LogInformation(
"Created SQS publisher for message type '{MessageType}' on queue '{QueueName}'.",
typeof(T),
writeConfiguration.QueueName);
}
[Then, Timeout(70000)] // ToDo: Sorry about this, but SQS is a little slow to verify against. Can be better I'm sure? ;)
public async Task QueueIsCreated()
{
var queue = new SqsQueueByName(RegionEndpoint.EUWest1,
QueueName, CreateMeABus.DefaultClientFactory().GetSqsClient(RegionEndpoint.EUWest1), 0);
await Patiently.AssertThatAsync(
queue.Exists, TimeSpan.FromSeconds(65));
}
public SqsQueueByName EnsureQueueExists(string region, SqsReadConfiguration queueConfig)
{
var sqsclient = AWSClientFactory.CreateAmazonSQSClient(RegionEndpoint.GetBySystemName(region));
var queue = new SqsQueueByName(queueConfig.QueueName, sqsclient, queueConfig.RetryCountBeforeSendingToErrorQueue);
queue.EnsureQueueAndErrorQueueExistAndAllAttributesAreUpdated(queueConfig);
return(queue);
}
[Then, Timeout(70000)] // ToDo: Sorry about this, but SQS is a little slow to verify against. Can be better I'm sure? ;)
public async Task QueueIsCreated()
{
var queue = new SqsQueueByName(RegionEndpoint.EUWest1,
QueueName, CreateMeABus.DefaultClientFactory().GetSqsClient(RegionEndpoint.EUWest1), 0);
await Patiently.AssertThatAsync(
queue.Exists, TimeSpan.FromSeconds(65));
}
[Then, Timeout(70000)] // ToDo: Sorry about this, but SQS is a little slow to verify against. Can be better I'm sure? ;)
public async Task QueueIsCreated()
{
var queue = new SqsQueueByName(RegionEndpoint.EUWest1,
QueueName, Client, 0, Substitute.For <ILoggerFactory>());
await Patiently.AssertThatAsync(
queue.Exists, TimeSpan.FromSeconds(65));
}
public async Task SqsPolicyWithAWildcardIsApplied()
{
var queue = new SqsQueueByName(RegionEndpoint.EUWest1, QueueName, Client, 0, Substitute.For <ILoggerFactory>());
await Patiently.AssertThatAsync(queue.Exists, TimeSpan.FromSeconds(60));
dynamic policyJson = JObject.Parse(queue.Policy);
policyJson.Statement.Count.ShouldBe(1, $"Expecting 1 statement in Sqs policy but found {policyJson.Statement.Count}");
}
public async Task SqsPolicyWithAWildcardIsApplied()
{
var queue = new SqsQueueByName(RegionEndpoint.EUWest1, QueueName, Client, 0);
await Patiently.AssertThatAsync(queue.Exists, TimeSpan.FromSeconds(60));
dynamic policyJson = JObject.Parse(queue.Policy);
Assert.IsTrue(policyJson.Statement.Count == 1, $"Expecting 1 statement in Sqs policy but found {policyJson.Statement.Count}");
}
public async Task SqsPolicyWithAWildcardIsApplied()
{
var queue = new SqsQueueByName(Region, QueueName, Client, 0, TestFixture.LoggerFactory);
await Patiently.AssertThatAsync(() => queue.ExistsAsync(), TimeSpan.FromSeconds(60));
dynamic policyJson = JObject.Parse(queue.Policy);
policyJson.Statement.Count.ShouldBe(1, $"Expecting 1 statement in Sqs policy but found {policyJson.Statement.Count}");
}
public async Task TagsAreNotAppliedIfNoneAreProvided()
{
// Arrange
var sut = new SqsQueueByName(RegionEndpoint.EUWest1, QueueName, _client, 3, NullLoggerFactory.Instance);
// Act
await sut.EnsureQueueAndErrorQueueExistAndAllAttributesAreUpdatedAsync(new SqsReadConfiguration(SubscriptionType.ToTopic), CancellationToken.None);
// Assert
await _client.Received(0).TagQueueAsync(Arg.Any <TagQueueRequest>());
}
public SqsQueueByName EnsureQueueExists(string region, SqsReadConfiguration queueConfig)
{
var regionEndpoint = RegionEndpoint.GetBySystemName(region);
var sqsclient = _awsClientFactory.GetAwsClientFactory().GetSqsClient(regionEndpoint);
var queue = _queueCache.TryGetFromCache(region, queueConfig.QueueName);
if (queue != null)
return queue;
queue = new SqsQueueByName(regionEndpoint, queueConfig.QueueName, sqsclient, queueConfig.RetryCountBeforeSendingToErrorQueue);
queue.EnsureQueueAndErrorQueueExistAndAllAttributesAreUpdated(queueConfig);
_queueCache.AddToCache(region, queue.QueueName, queue);
return queue;
}
public SqsQueueByName EnsureQueueExists(string region, SqsReadConfiguration queueConfig)
{
var regionEndpoint = RegionEndpoint.GetBySystemName(region);
var sqsclient = _awsClientFactory.GetAwsClientFactory().GetSqsClient(regionEndpoint);
var queue = _queueCache.TryGetFromCache(region, queueConfig.QueueName);
if (queue != null)
{
return(queue);
}
queue = new SqsQueueByName(regionEndpoint, queueConfig.QueueName, sqsclient, queueConfig.RetryCountBeforeSendingToErrorQueue);
queue.EnsureQueueAndErrorQueueExistAndAllAttributesAreUpdated(queueConfig);
_queueCache.AddToCache(region, queue.QueueName, queue);
return(queue);
}
private List<Message> PopMessagesFromSourceQueue(SqsQueueByName sourceQueue)
{
var messages = new List<Message>();
ReceiveMessageResponse receiveResponse;
do
{
receiveResponse = sourceQueue.Client.ReceiveMessage(new ReceiveMessageRequest
{
QueueUrl = sourceQueue.Url,
MaxNumberOfMessages = Count,
});
messages.AddRange(receiveResponse.Messages);
} while (messages.Count < Count && receiveResponse.Messages.Any());
return messages;
}
protected override void Given()
{
int retryCount = 1;
_client = Substitute.For <IAmazonSQS>();
var response = GenerateResponseMessage(MessageTypeString, Guid.NewGuid());
_client.ReceiveMessageAsync(
Arg.Any <ReceiveMessageRequest>(),
Arg.Any <CancellationToken>())
.Returns(
x => Task.FromResult(response),
x => Task.FromResult(new ReceiveMessageResponse()));
_client.GetQueueUrlAsync(Arg.Any <string>())
.Returns(x =>
{
if (x.Arg <string>() == "some-queue-name")
{
return new GetQueueUrlResponse
{
QueueUrl = "https://testqueues.com/some-queue-name"
}
}
;
throw new QueueDoesNotExistException("some-queue-name not found");
});
_client.GetQueueAttributesAsync(Arg.Any <GetQueueAttributesRequest>())
.Returns(new GetQueueAttributesResponse()
{
Attributes = new Dictionary <string, string> {
{ "QueueArn", "something:some-queue-name" }
}
});
var queue = new SqsQueueByName(RegionEndpoint.EUWest1, "some-queue-name", _client, retryCount, LoggerFactory);
queue.ExistsAsync().Wait();
_queue = queue;
Queues.Add(_queue);
Handler.Handle(null)
.ReturnsForAnyArgs(true).AndDoes(ci => Interlocked.Increment(ref _callCount));
}
public QueueWithAsyncStartup <SqsQueueByName> EnsureQueueExists(
string region,
SqsReadConfiguration queueConfig)
{
var regionEndpoint = RegionEndpoint.GetBySystemName(region);
var sqsclient = _awsClientFactory.GetAwsClientFactory().GetSqsClient(regionEndpoint);
var queue = new SqsQueueByName(regionEndpoint,
queueConfig.QueueName,
sqsclient,
queueConfig.RetryCountBeforeSendingToErrorQueue,
_loggerFactory);
var startupTask = queue.EnsureQueueAndErrorQueueExistAndAllAttributesAreUpdatedAsync(queueConfig);
// This startupTask is intentionally not awaited, as it will be run when the bus is started.
return(new QueueWithAsyncStartup <SqsQueueByName>(startupTask, queue));
}
private List <Message> PopMessagesFromSourceQueue(SqsQueueByName sourceQueue)
{
var messages = new List <Message>();
ReceiveMessageResponse receiveResponse;
do
{
receiveResponse = sourceQueue.Client.ReceiveMessage(new ReceiveMessageRequest
{
QueueUrl = sourceQueue.Uri.AbsoluteUri,
MaxNumberOfMessages = Count
});
messages.AddRange(receiveResponse.Messages);
} while (messages.Count < Count && receiveResponse.Messages.Any());
return(messages);
}
public QueueWithAsyncStartup EnsureQueueExists(
string region,
SqsReadConfiguration queueConfig)
{
var regionEndpoint = RegionEndpoint.GetBySystemName(region);
var sqsClient = _awsClientFactory.GetAwsClientFactory().GetSqsClient(regionEndpoint);
#pragma warning disable 618
var queue = new SqsQueueByName(regionEndpoint,
queueConfig.QueueName,
sqsClient,
queueConfig.RetryCountBeforeSendingToErrorQueue,
_loggerFactory);
#pragma warning restore 618
var startupTask = new Func <Task>(() => queue.EnsureQueueAndErrorQueueExistAndAllAttributesAreUpdatedAsync(queueConfig));
return(new QueueWithAsyncStartup(startupTask, queue));
}
public async Task TagsAreAppliedToParentAndErrorQueues()
{
// Arrange
var sut = new SqsQueueByName(RegionEndpoint.EUWest1, QueueName, _client, 3, NullLoggerFactory.Instance);
var config = new SqsReadConfiguration(SubscriptionType.ToTopic)
{
Tags = new Dictionary <string, string>
{
["TagOne"] = "tag-one",
["TagTwo"] = "tag-two"
}
};
// Act
await sut.EnsureQueueAndErrorQueueExistAndAllAttributesAreUpdatedAsync(config, CancellationToken.None);
// Assert
await _client.Received(1).TagQueueAsync(Arg.Is <TagQueueRequest>(req => req.QueueUrl == QueueUrl && req.Tags == config.Tags));
}
public async Task <bool> ExecuteAsync()
{
Console.WriteLine($"Moving {Count} messages from {SourceQueueName} to {DestinationQueueName} in {Region}.");
var loggerFactory = new LoggerFactory();
var config = new AmazonSQSConfig {
RegionEndpoint = RegionEndpoint.GetBySystemName(Region)
};
var client = new DefaultAwsClientFactory().GetSqsClient(config.RegionEndpoint);
var sourceQueue = new SqsQueueByName(config.RegionEndpoint, SourceQueueName, client, JustSayingConstants.DefaultHandlerRetryCount, loggerFactory);
var destinationQueue = new SqsQueueByName(config.RegionEndpoint, DestinationQueueName, client, JustSayingConstants.DefaultHandlerRetryCount, loggerFactory);
await EnsureQueueExistsAsync(sourceQueue).ConfigureAwait(false);
await EnsureQueueExistsAsync(destinationQueue).ConfigureAwait(false);
var messages = PopMessagesFromSourceQueue(sourceQueue);
var receiptHandles = messages.ToDictionary(m => m.MessageId, m => m.ReceiptHandle);
var sendResponse = destinationQueue.Client.SendMessageBatch(new SendMessageBatchRequest
{
QueueUrl = destinationQueue.Uri.AbsoluteUri,
Entries = messages.Select(x => new SendMessageBatchRequestEntry {
Id = x.MessageId, MessageBody = x.Body
}).ToList()
});
sourceQueue.Client.DeleteMessageBatch(new DeleteMessageBatchRequest
{
QueueUrl = sourceQueue.Uri.AbsoluteUri,
Entries = sendResponse.Successful.Select(x => new DeleteMessageBatchRequestEntry
{
Id = x.Id,
ReceiptHandle = receiptHandles[x.Id]
}).ToList()
});
Console.WriteLine($"Moved {sendResponse.Successful.Count} messages from {SourceQueueName} to {DestinationQueueName} in {Region}.");
return(true);
}
protected override void Given()
{
int retryCount = 1;
_client = new FakeAmazonSqs(() =>
{
return(new[] { GenerateResponseMessages(MessageTypeString, Guid.NewGuid()) }
.Concat(new ReceiveMessageResponse().Infinite()));
});
var queue = new SqsQueueByName(RegionEndpoint.EUWest1,
"some-queue-name",
_client,
retryCount,
LoggerFactory);
queue.ExistsAsync().Wait();
_queue = queue;
Queues.Add(_queue);
}
public async Task QueueIsCreated()
{
async Task QueueIsCreatedInner()
{
var queue = new SqsQueueByName(RegionEndpoint.EUWest1,
QueueName, Client, 0, Substitute.For <ILoggerFactory>());
await Patiently.AssertThatAsync(
queue.Exists, TimeSpan.FromSeconds(65));
}
var task = QueueIsCreatedInner();
if (task == await Task.WhenAny(task, Task.Delay(TimeSpan.FromSeconds(70)))) // ToDo: Sorry about this, but SQS is a little slow to verify against. Can be better I'm sure? ;)
{
await task;
}
else
{
throw new TimeoutException();
}
}
public async Task Then_The_Error_Queue_Is_Created()
{
// Arrange
ILoggerFactory loggerFactory = OutputHelper.ToLoggerFactory();
IAwsClientFactory clientFactory = CreateClientFactory();
var client = clientFactory.GetSqsClient(Region);
var queue = new SqsQueueByName(
Region,
UniqueName,
client,
1,
loggerFactory);
// Act
await queue.CreateAsync(new SqsBasicConfiguration());
// Assert
await Patiently.AssertThatAsync(
async() => await queue.ErrorQueue.ExistsAsync(),
40.Seconds());
}
public async Task HandlingManyMessages(int throttleMessageCount)
{
// Arrange
var fixture = new JustSayingFixture(OutputHelper);
var client = fixture.CreateSqsClient();
var queue = new SqsQueueByName(fixture.Region, fixture.UniqueName, client, 1, fixture.LoggerFactory);
if (!await queue.ExistsAsync())
{
await queue.CreateAsync(new SqsBasicConfiguration());
if (!fixture.IsSimulator)
{
// Wait for up to 60 secs for queue creation to be guaranteed completed by AWS
using (var cts = new CancellationTokenSource(TimeSpan.FromMinutes(1)))
{
while (!cts.IsCancellationRequested)
{
await Task.Delay(TimeSpan.FromSeconds(2));
if (await queue.ExistsAsync())
{
break;
}
}
}
}
}
Assert.True(await queue.ExistsAsync(), "The queue was not created.");
OutputHelper.WriteLine($"{DateTime.Now} - Adding {throttleMessageCount} messages to the queue.");
var entriesAdded = 0;
// Add some messages
do
{
var entries = new List <SendMessageBatchRequestEntry>();
for (var j = 0; j < 10; j++)
{
var batchEntry = new SendMessageBatchRequestEntry
{
MessageBody = $"{{\"Subject\":\"GenericMessage\", \"Message\": \"{entriesAdded}\"}}",
Id = Guid.NewGuid().ToString()
};
entries.Add(batchEntry);
entriesAdded++;
}
await client.SendMessageBatchAsync(queue.Url, entries);
}while (entriesAdded < throttleMessageCount);
OutputHelper.WriteLine($"{DateTime.Now} - Done adding messages.");
var handleCount = 0;
var serialisations = Substitute.For <IMessageSerialisationRegister>();
var monitor = Substitute.For <IMessageMonitor>();
var handler = Substitute.For <IHandlerAsync <SimpleMessage> >();
handler.Handle(null).ReturnsForAnyArgs(true).AndDoes(_ => Interlocked.Increment(ref handleCount));
serialisations.DeserializeMessage(string.Empty).ReturnsForAnyArgs(new SimpleMessage());
var listener = new SqsNotificationListener(queue, serialisations, monitor, fixture.LoggerFactory);
listener.AddMessageHandler(() => handler);
// Act
var stopwatch = Stopwatch.StartNew();
listener.Listen();
using (var cts = new CancellationTokenSource(TimeSpan.FromMinutes(5)))
{
do
{
if (!fixture.IsSimulator)
{
await Task.Delay(TimeSpan.FromSeconds(5));
}
OutputHelper.WriteLine($"{DateTime.Now} - Handled {handleCount} messages. Waiting for completion.");
}while (handleCount < throttleMessageCount && !cts.IsCancellationRequested);
}
listener.StopListening();
stopwatch.Stop();
OutputHelper.WriteLine($"{DateTime.Now} - Handled {handleCount:N0} messages.");
OutputHelper.WriteLine($"{DateTime.Now} - Took {stopwatch.ElapsedMilliseconds} ms");
OutputHelper.WriteLine($"{DateTime.Now} - Throughput {(float)handleCount / stopwatch.ElapsedMilliseconds * 1000} messages/second");
// Assert
Assert.Equal(throttleMessageCount, handleCount);
}
private void EnsureQueueExists(SqsQueueByName queue)
{
if (!queue.Exists()) throw new System.InvalidOperationException($"{queue.QueueName} does not exist.");
}
// Use this to manually test the performance / throttling of getting messages out of the queue.
public void HandlingManyMessages(int throttleMessageCount)
{
var locker = new object();
var awsQueueClient = CreateMeABus.DefaultClientFactory().GetSqsClient(RegionEndpoint.EUWest1);
var q = new SqsQueueByName(RegionEndpoint.EUWest1, "throttle_test", awsQueueClient, 1);
if (!q.Exists())
{
q.Create(new SqsBasicConfiguration());
Thread.Sleep(TimeSpan.FromMinutes(1)); // wait 60 secs for queue creation to be guaranteed completed by aws. :(
}
Assert.True(q.Exists());
Console.WriteLine("{0} - Adding {1} messages to the queue.", DateTime.Now, throttleMessageCount);
var entriesAdded = 0;
// Add some messages
do
{
var entries = new List<SendMessageBatchRequestEntry>();
for (var j = 0; j < 10; j++)
{
var batchEntry = new SendMessageBatchRequestEntry
{
MessageBody = "{\"Subject\":\"GenericMessage\", \"Message\": \"" + entriesAdded.ToString() + "\"}",
Id = Guid.NewGuid().ToString()
};
entries.Add(batchEntry);
entriesAdded++;
}
awsQueueClient.SendMessageBatch(new SendMessageBatchRequest { QueueUrl = q.Url, Entries = entries });
}
while (entriesAdded < throttleMessageCount);
Console.WriteLine("{0} - Done adding messages.", DateTime.Now);
var handleCount = 0;
var serialisations = Substitute.For<IMessageSerialisationRegister>();
var monitor = Substitute.For<IMessageMonitor>();
var handler = Substitute.For<IHandler<GenericMessage>>();
handler.Handle(null).ReturnsForAnyArgs(true).AndDoes(x => {lock (locker) { handleCount++; } });
serialisations.DeserializeMessage(string.Empty).ReturnsForAnyArgs(new GenericMessage());
var listener = new SqsNotificationListener(q, serialisations, monitor);
listener.AddMessageHandler(() => handler);
var stopwatch = new Stopwatch();
stopwatch.Start();
listener.Listen();
var waitCount = 0;
do
{
Thread.Sleep(TimeSpan.FromSeconds(5));
Console.WriteLine("{0} - Handled {1} messages. Waiting for completion.", DateTime.Now, handleCount);
waitCount++;
}
while (handleCount < throttleMessageCount && waitCount < 100);
listener.StopListening();
stopwatch.Stop();
Console.WriteLine("{0} - Handled {1} messages.", DateTime.Now, handleCount);
Console.WriteLine("{0} - Took {1} ms", DateTime.Now, stopwatch.ElapsedMilliseconds);
Console.WriteLine("{0} - Throughput {1} msg/sec", DateTime.Now, (float)handleCount / stopwatch.ElapsedMilliseconds * 1000);
Assert.AreEqual(throttleMessageCount, handleCount);
}
private void EnsureQueueIsSubscribedToTopic(RegionEndpoint region, SnsTopicByName eventTopic, SqsQueueByName queue)
{
var sqsclient = _awsClientFactory.GetAwsClientFactory().GetSqsClient(region);
eventTopic.Subscribe(sqsclient, queue);
}
public void IncorrectPartialQueueNameDoNotMatch()
{
var sqsQueueByName = new SqsQueueByName(RegionEndpoint.EUWest1, "some-queue", _client, RetryCount);
Assert.IsFalse(sqsQueueByName.Exists());
}
private static void EnsureQueueIsSubscribedToTopic(string region, SnsTopicByName eventTopic, SqsQueueByName queue)
{
var sqsclient = AWSClientFactory.CreateAmazonSQSClient(RegionEndpoint.GetBySystemName(region));
eventTopic.Subscribe(sqsclient, queue);
}
public void CorrectQueueNameShouldMatch()
{
var sqsQueueByName = new SqsQueueByName("some-queue-name", _client, RetryCount);
Assert.IsTrue(sqsQueueByName.Exists());
}
public void IncorrectQueueNameDoNotMatch()
{
var sqsQueueByName = new SqsQueueByName("some-queue-name1", _client, RetryCount);
Assert.IsFalse(sqsQueueByName.Exists());
}
