public async Task FixIssueOfDLQBeingCleared()
{
var sqsClient = new AmazonSQSClient(RegionEndpoint.USEast2);
var queueUrl = (await sqsClient.CreateQueueAsync("lambda-test-" + DateTime.Now.Ticks)).QueueUrl;
var queueArn = (await sqsClient.GetQueueAttributesAsync(queueUrl, new List <string> {
"QueueArn"
})).QueueARN;
try
{
var assembly = this.GetType().GetTypeInfo().Assembly;
var fullPath = Path.GetFullPath(Path.GetDirectoryName(assembly.Location) + "../../../../../../testapps/TestFunction");
var initialDeployCommand = new DeployFunctionCommand(new TestToolLogger(_testOutputHelper), fullPath, new string[0]);
initialDeployCommand.FunctionName = "test-function-" + DateTime.Now.Ticks;
initialDeployCommand.Handler = "TestFunction::TestFunction.Function::ToUpper";
initialDeployCommand.Timeout = 10;
initialDeployCommand.MemorySize = 512;
initialDeployCommand.Role = await TestHelper.GetTestRoleArnAsync();
initialDeployCommand.Configuration = "Release";
initialDeployCommand.TargetFramework = "netcoreapp2.1";
initialDeployCommand.Runtime = "dotnetcore2.1";
initialDeployCommand.DeadLetterTargetArn = queueArn;
initialDeployCommand.DisableInteractive = true;
var created = await initialDeployCommand.ExecuteAsync();
try
{
Assert.True(created);
var funcConfig = await initialDeployCommand.LambdaClient.GetFunctionConfigurationAsync(initialDeployCommand.FunctionName);
Assert.Equal(queueArn, funcConfig.DeadLetterConfig?.TargetArn);
var redeployCommand = new DeployFunctionCommand(new TestToolLogger(_testOutputHelper), fullPath, new string[0]);
redeployCommand.FunctionName = initialDeployCommand.FunctionName;
redeployCommand.Configuration = "Release";
redeployCommand.TargetFramework = "netcoreapp2.1";
redeployCommand.Runtime = "dotnetcore2.1";
redeployCommand.DisableInteractive = true;
var redeployed = await redeployCommand.ExecuteAsync();
Assert.True(redeployed);
funcConfig = await initialDeployCommand.LambdaClient.GetFunctionConfigurationAsync(initialDeployCommand.FunctionName);
Assert.Equal(queueArn, funcConfig.DeadLetterConfig?.TargetArn);
redeployCommand = new DeployFunctionCommand(new TestToolLogger(_testOutputHelper), fullPath, new string[0]);
redeployCommand.FunctionName = initialDeployCommand.FunctionName;
redeployCommand.Configuration = "Release";
redeployCommand.TargetFramework = "netcoreapp2.1";
redeployCommand.Runtime = "dotnetcore2.1";
redeployCommand.DeadLetterTargetArn = "";
redeployCommand.DisableInteractive = true;
redeployed = await redeployCommand.ExecuteAsync();
Assert.True(redeployed);
funcConfig = await initialDeployCommand.LambdaClient.GetFunctionConfigurationAsync(initialDeployCommand.FunctionName);
Assert.Null(funcConfig.DeadLetterConfig?.TargetArn);
}
finally
{
if (created)
{
await initialDeployCommand.LambdaClient.DeleteFunctionAsync(initialDeployCommand.FunctionName);
}
}
}
finally
{
await sqsClient.DeleteQueueAsync(queueUrl);
}
}
public async Task CreateQueueIfNotExistsAsync(string queueName)
{
var createQueueRequest = new CreateQueueRequest(queueName);
await _client.CreateQueueAsync(createQueueRequest);
}
public async Task DlqIntegTest()
{
if (!TestUtils.ProfileTestsEnabled)
{
return;
}
const int WAIT_TIME = 5000;
var queueName = "local-dlq-list-queue-test-" + DateTime.Now.Ticks;
using (var client = new AmazonSQSClient(TestUtils.GetAWSCredentials(), TestUtils.TestRegion))
{
var createResponse = await client.CreateQueueAsync(new CreateQueueRequest { QueueName = queueName });
await TestUtils.WaitTillQueueIsCreatedAsync(client, createResponse.QueueUrl);
try
{
var configFile = TestUtils.GetLambdaFunctionSourceFile("ToUpperFunc", "aws-lambda-tools-defaults.json");
var buildPath = TestUtils.GetLambdaFunctionBuildPath("ToUpperFunc");
var configInfo = LambdaDefaultsConfigFileParser.LoadFromFile(configFile);
var runtime = LocalLambdaRuntime.Initialize(buildPath);
var function = runtime.LoadLambdaFunctions(configInfo.FunctionInfos)[0];
var monitor = new DlqMonitor(runtime, function, TestUtils.TestProfile, TestUtils.TestRegion.SystemName, createResponse.QueueUrl);
monitor.Start();
await client.SendMessageAsync(new SendMessageRequest
{
QueueUrl = createResponse.QueueUrl,
MessageBody = "\"testing dlq\""
});
Thread.Sleep(WAIT_TIME);
var logs = monitor.FetchNewLogs();
Assert.Single(logs);
Assert.Contains("testing dlq", logs[0].Logs);
Assert.NotNull(logs[0].ReceiptHandle);
Assert.NotEqual(DateTime.MinValue, logs[0].ProcessTime);
logs = monitor.FetchNewLogs();
Assert.Equal(0, logs.Count);
await client.SendMessageAsync(new SendMessageRequest
{
QueueUrl = createResponse.QueueUrl,
MessageBody = "\"testing dlq1\""
});
await client.SendMessageAsync(new SendMessageRequest
{
QueueUrl = createResponse.QueueUrl,
MessageBody = "\"testing dlq2\""
});
Thread.Sleep(WAIT_TIME);
logs = monitor.FetchNewLogs();
Assert.Equal(2, logs.Count);
monitor.Stop();
Thread.Sleep(WAIT_TIME);
await client.SendMessageAsync(new SendMessageRequest
{
QueueUrl = createResponse.QueueUrl,
MessageBody = "\"testing dlq3\""
});
Thread.Sleep(WAIT_TIME);
logs = monitor.FetchNewLogs();
Assert.Equal(0, logs.Count);
}
finally
{
await client.DeleteQueueAsync(createResponse.QueueUrl);
}
}
}
private void CreateQueue(AmazonSQSClient sqsClient)
{
_logger.Value.Debug($"Queue does not exist, creating queue: {_subscription.ChannelName.Value} subscribed to {_subscription.RoutingKey.Value} on {_awsConnection.Region}");
_queueUrl = null;
try
{
var attributes = new Dictionary <string, string>();
if (_subscription.RedrivePolicy != null && _dlqARN != null)
{
var policy = new { maxReceiveCount = _subscription.RedrivePolicy.MaxReceiveCount, deadLetterTargetArn = _dlqARN };
attributes.Add("RedrivePolicy", JsonConvert.SerializeObject(policy));
}
attributes.Add("DelaySeconds", _subscription.DelaySeconds.ToString());
attributes.Add("MessageRetentionPeriod", _subscription.MessageRetentionPeriod.ToString());
if (_subscription.IAMPolicy != null)
{
attributes.Add("Policy", _subscription.IAMPolicy);
}
attributes.Add("ReceiveMessageWaitTimeSeconds", ToSecondsAsString(_subscription.TimeoutInMiliseconds));
attributes.Add("VisibilityTimeout", _subscription.LockTimeout.ToString());
var tags = new Dictionary <string, string>();
tags.Add("Source", "Brighter");
if (_subscription.Tags != null)
{
foreach (var tag in _subscription.Tags)
{
tags.Add(tag.Key, tag.Value);
}
}
var request = new CreateQueueRequest(_subscription.ChannelName.Value)
{
Attributes = attributes,
Tags = tags
};
var response = sqsClient.CreateQueueAsync(request).GetAwaiter().GetResult();
_queueUrl = response.QueueUrl;
if (!string.IsNullOrEmpty(_queueUrl))
{
_logger.Value.Debug($"Queue created: {_queueUrl}");
using (var snsClient = new AmazonSimpleNotificationServiceClient(_awsConnection.Credentials, _awsConnection.Region))
{
CheckSubscription(_subscription.MakeChannels, sqsClient, snsClient);
}
}
else
{
throw new InvalidOperationException($"Could not create queue: {_subscription.ChannelName.Value} subscribed to {_channelTopicArn} on {_awsConnection.Region}");
}
}
catch (QueueDeletedRecentlyException ex)
{
//QueueDeletedRecentlyException - wait 30 seconds then retry
//Although timeout is 60s, we could be partway through that, so apply Copernican Principle
//and assume we are halfway through
var error = $"Could not create queue {_subscription.ChannelName.Value} because {ex.Message} waiting 60s to retry";
_logger.Value.Error(error);
Thread.Sleep(TimeSpan.FromSeconds(30));
throw new ChannelFailureException(error, ex);
}
catch (AmazonSQSException ex)
{
var error = $"Could not create queue {_queueUrl} subscribed to topic {_subscription.RoutingKey.Value} in region {_awsConnection.Region.DisplayName} because {ex.Message}";
_logger.Value.Error(error);
throw new InvalidOperationException(error, ex);
}
catch (HttpErrorResponseException ex)
{
var error = $"Could not create queue {_queueUrl} subscribed to topic {_subscription.RoutingKey.Value} in region {_awsConnection.Region.DisplayName} because {ex.Message}";
_logger.Value.Error(error);
throw new InvalidOperationException(error, ex);
}
}
public async override Task OpenAsync(string correlationId, ConnectionParams connection, CredentialParams credential)
{
var awsConnection = new AwsConnectionParams(connection, credential);
// Assign service name
awsConnection.Service = "sqs";
// Assign queue name
var queueName = awsConnection.Resource ?? awsConnection.Get("queue") ?? Name;
awsConnection.Resource = queueName;
var deadQueueName = awsConnection.Get("dead_queue");
// Determine if a fifo queue is being used
_fifoQueue = queueName.EndsWith(".fifo", StringComparison.OrdinalIgnoreCase) ? true : false;
// Validate connection params
var err = awsConnection.Validate(correlationId);
if (err != null)
{
throw err;
}
_logger.Info(null, "Connecting queue {0} to {1}", Name, awsConnection.Arn);
var region = RegionEndpoint.GetBySystemName(awsConnection.Region);
var config = new AmazonSQSConfig()
{
RegionEndpoint = region,
UseHttp = true
};
_client = new AmazonSQSClient(awsConnection.AccessId, awsConnection.AccessKey, config);
try
{
try
{
// Create queue if it doesn't exist
var queueRequest = new CreateQueueRequest(queueName);
if (_fifoQueue)
{
queueRequest.Attributes.Add("FifoQueue", "true");
}
await _client.CreateQueueAsync(queueRequest);
}
catch (QueueNameExistsException)
{
// Ignore exception.
}
try
{
// Create dead queue if it doesn't exist
if (!string.IsNullOrEmpty(deadQueueName))
{
var deadQueueRequest = new CreateQueueRequest(deadQueueName);
if (_fifoQueue)
{
deadQueueRequest.Attributes.Add("FifoQueue", "true");
}
await _client.CreateQueueAsync(deadQueueRequest);
}
}
catch (QueueNameExistsException)
{
// Ignore exception.
}
var response = await _client.GetQueueUrlAsync(queueName);
_queue = response.QueueUrl;
if (_fifoQueue)
{
var temp = await GetQueueAttributesAsync(correlationId, new List <string>() { "ContentBasedDeduplication" }, _queue);
_contentBasedDupication = BooleanConverter.ToBooleanWithDefault(temp["ContentBasedDeduplication"], _contentBasedDupication);
}
if (!string.IsNullOrEmpty(deadQueueName))
{
response = await _client.GetQueueUrlAsync(deadQueueName);
_deadQueue = response.QueueUrl;
if (_fifoQueue)
{
var temp = await GetQueueAttributesAsync(correlationId, new List <string>() { "ContentBasedDeduplication" }, _deadQueue);
_contentBasedDupicationDlq = BooleanConverter.ToBooleanWithDefault(temp["ContentBasedDeduplication"], _contentBasedDupicationDlq);
}
}
else
{
_deadQueue = null;
}
}
catch (Exception ex)
{
throw new ConnectionException(correlationId, "CANNOT_ACCESS_QUEUE", "Failed to access SQS queue", ex)
.WithDetails("queue", _queue);
}
}
private void EnsureQueue(Connection connection)
{
using (var sqsClient = new AmazonSQSClient(_awsConnection.Credentials, _awsConnection.Region))
{
//Does the queue exist - this is an HTTP call, we should cache the results for a period of time
(bool exists, string name)queueExists = QueueExists(sqsClient, connection.ChannelName.ToValidSQSQueueName());
if (!queueExists.exists)
{
try
{
var request = new CreateQueueRequest(connection.ChannelName.ToValidSQSQueueName())
{
Attributes =
{
{ "VisibilityTimeout", connection.VisibilityTimeout.ToString() },
{ "ReceiveMessageWaitTimeSeconds", ToSecondsAsString(connection.TimeoutInMiliseconds) }
}
};
var response = sqsClient.CreateQueueAsync(request).Result;
var queueUrl = response.QueueUrl;
if (!string.IsNullOrEmpty(queueUrl))
{
//topic might not exist
using (var snsClient = new AmazonSimpleNotificationServiceClient(_awsConnection.Credentials, _awsConnection.Region))
{
var exists = snsClient.ListTopicsAsync().Result.Topics.SingleOrDefault(topic => topic.TopicArn == connection.RoutingKey);
if (exists == null)
{
var createTopic = snsClient.CreateTopicAsync(new CreateTopicRequest(connection.RoutingKey.ToValidSNSTopicName())).Result;
if (!string.IsNullOrEmpty(createTopic.TopicArn))
{
var subscription = snsClient.SubscribeQueueAsync(createTopic.TopicArn, sqsClient, queueUrl).Result;
//We need to support raw messages to allow the use of message attributes
snsClient.SetSubscriptionAttributesAsync(new SetSubscriptionAttributesRequest(subscription, "RawMessageDelivery", "true"));
}
}
}
}
}
catch (AggregateException ae)
{
//TODO: We need some retry semantics here
//TODO: We need to flatten the ae and handle some of these with ae.Handle((x) => {})
ae.Handle(ex =>
{
if (ex is QueueDeletedRecentlyException)
{
//QueueDeletedRecentlyException - wait 30 seconds then retry
//Although timeout is 60s, we could be partway through that, so apply Copernican Principle
//and assume we are halfway through
var error = $"Could not create queue {connection.ChannelName.ToValidSQSQueueName()} because {ae.Message} waiting 60s to retry";
_logger.Value.Error(error);
Task.Delay(TimeSpan.FromSeconds(30));
throw new ChannelFailureException(error, ae);
}
return(false);
});
}
}
}
}
static async void MainAsync(string[] args)
{
try
{
Console.WriteLine("===========================================");
Console.WriteLine("Getting Started with Amazon SQS");
Console.WriteLine("===========================================\n");
//Creating a queue
Console.WriteLine("Create a queue called MyQueue.\n");
var sqsRequest = new CreateQueueRequest {
QueueName = "MyQueue11"
};
var createQueueResponse = await _sqs.CreateQueueAsync(sqsRequest);
_myQueueUrl = createQueueResponse.QueueUrl;
//Confirming the queue exists
var listQueuesRequest = new ListQueuesRequest();
var listQueuesResponse = await _sqs.ListQueuesAsync(listQueuesRequest);
Console.WriteLine("Printing list of Amazon SQS queues.\n");
if (listQueuesResponse.QueueUrls != null)
{
foreach (String queueUrl in listQueuesResponse.QueueUrls)
{
Console.WriteLine(" QueueUrl: {0}", queueUrl);
}
}
Console.WriteLine();
//Sending a message
for (int i = 0; i < 10; i++)
{
var message = $"This is my message text-Id-{Guid.NewGuid().ToString("N")}";
//var message = $"This is my message text";
Console.WriteLine($"Sending a message to MyQueue : {message}");
var sendMessageRequest = new SendMessageRequest
{
QueueUrl = _myQueueUrl, //URL from initial queue creation
MessageBody = message
};
await _sqs.SendMessageAsync(sendMessageRequest);
}
//start of 5 receiver tasks
var tasks = Enumerable.Range(0, 5).Select(number =>
Task.Run(async() =>
await ReceiveMessage(number)
)).ToList();
await Task.WhenAll(tasks);
}
catch (AmazonSQSException ex)
{
Console.WriteLine("Caught Exception: " + ex.Message);
Console.WriteLine("Response Status Code: " + ex.StatusCode);
Console.WriteLine("Error Code: " + ex.ErrorCode);
Console.WriteLine("Error Type: " + ex.ErrorType);
Console.WriteLine("Request ID: " + ex.RequestId);
}
Console.WriteLine("Press Enter to continue...");
Console.Read();
}
public async Task SetQueueConfigurationTests()
{
var filterRule = new FilterRule("Prefix", "test/");
using (var sqsClient = new AmazonSQSClient())
{
string topicName = UtilityMethods.GenerateName("events-test");
var createResponse = await sqsClient.CreateQueueAsync(topicName);
var bucketName = await UtilityMethods.CreateBucketAsync(Client, "SetQueueConfigurationTests");
try
{
var queueArn = await sqsClient.AuthorizeS3ToSendMessageAsync(createResponse.QueueUrl, bucketName);
PutBucketNotificationRequest putRequest = new PutBucketNotificationRequest
{
BucketName = bucketName,
QueueConfigurations = new List <QueueConfiguration>
{
new QueueConfiguration
{
Id = "the-queue-test",
Queue = queueArn,
Events = { EventType.ObjectCreatedPut },
Filter = new Filter
{
S3KeyFilter = new S3KeyFilter
{
FilterRules = new List <FilterRule>
{
filterRule
}
}
}
}
}
};
await Client.PutBucketNotificationAsync(putRequest);
var getResponse = WaitUtils.WaitForComplete(
() =>
{
return(Client.GetBucketNotificationAsync(bucketName).Result);
},
(r) =>
{
return(r.QueueConfigurations.Count > 0);
});
Assert.Equal(1, getResponse.QueueConfigurations.Count);
Assert.Equal(1, getResponse.QueueConfigurations[0].Events.Count);
Assert.Equal(EventType.ObjectCreatedPut, getResponse.QueueConfigurations[0].Events[0]);
Assert.NotNull(getResponse.QueueConfigurations[0].Filter);
Assert.NotNull(getResponse.QueueConfigurations[0].Filter.S3KeyFilter);
Assert.NotNull(getResponse.QueueConfigurations[0].Filter.S3KeyFilter.FilterRules);
Assert.Equal(1, getResponse.QueueConfigurations[0].Filter.S3KeyFilter.FilterRules.Count);
Assert.Equal(filterRule.Name, getResponse.QueueConfigurations[0].Filter.S3KeyFilter.FilterRules[0].Name);
Assert.Equal(filterRule.Value, getResponse.QueueConfigurations[0].Filter.S3KeyFilter.FilterRules[0].Value);
Assert.Equal("the-queue-test", getResponse.QueueConfigurations[0].Id);
Assert.Equal(queueArn, getResponse.QueueConfigurations[0].Queue);
// Purge queue to remove test message sent configuration was setup.
await sqsClient.PurgeQueueAsync(createResponse.QueueUrl);
// make sure the queue is really clear
WaitUtils.WaitForComplete(
() =>
{
return(sqsClient.ReceiveMessageAsync(new ReceiveMessageRequest()
{
QueueUrl = createResponse.QueueUrl
}).Result);
},
(r) =>
{
return(r.Messages.Count == 0);
});
var putObjectRequest = new PutObjectRequest
{
BucketName = bucketName,
Key = "test/data.txt",
ContentBody = "Important Data"
};
await Client.PutObjectAsync(putObjectRequest);
var response = WaitUtils.WaitForComplete(
() =>
{
return(sqsClient.ReceiveMessageAsync(new ReceiveMessageRequest
{
QueueUrl = createResponse.QueueUrl,
WaitTimeSeconds = 20
}).Result);
},
(r) =>
{
return(r.Messages.Count > 0);
});
var evnt = S3EventNotification.ParseJson(response.Messages[0].Body);
Assert.Equal(1, evnt.Records.Count);
Assert.Equal(putObjectRequest.BucketName, evnt.Records[0].S3.Bucket.Name);
Assert.Equal(putObjectRequest.Key, evnt.Records[0].S3.Object.Key);
Assert.Equal(putObjectRequest.ContentBody.Length, evnt.Records[0].S3.Object.Size);
}
finally
{
await sqsClient.DeleteQueueAsync(createResponse.QueueUrl);
await UtilityMethods.DeleteBucketWithObjectsAsync(Client, bucketName);
}
}
}
private async Task CreateQueue()
{
var result = await sqsClient.CreateQueueAsync("MyQueue");
queueUrl = result.QueueUrl;
}
private void CreateDLQ(AmazonSQSClient sqsClient)
{
try
{
var request = new CreateQueueRequest(_subscription.RedrivePolicy.DeadlLetterQueueName.Value);
var createDeadLetterQueueResponse = sqsClient.CreateQueueAsync(request).GetAwaiter().GetResult();
var queueUrl = createDeadLetterQueueResponse.QueueUrl;
if (!string.IsNullOrEmpty(queueUrl))
{
//We need the ARN of the dead letter queue to configure the queue redrive policy, not the name
var attributesRequest = new GetQueueAttributesRequest
{
QueueUrl = queueUrl,
AttributeNames = new List <string> {
"QueueArn"
}
};
var attributesResponse = sqsClient.GetQueueAttributesAsync(attributesRequest).GetAwaiter().GetResult();
if (attributesResponse.HttpStatusCode != HttpStatusCode.OK)
{
throw new InvalidOperationException($"Could not find ARN of DLQ, status: {attributesResponse.HttpStatusCode}");
}
_dlqARN = attributesResponse.QueueARN;
}
else
{
throw new InvalidOperationException($"Could not find create DLQ, status: {createDeadLetterQueueResponse.HttpStatusCode}");
}
}
catch (QueueDeletedRecentlyException ex)
{
//QueueDeletedRecentlyException - wait 30 seconds then retry
//Although timeout is 60s, we could be partway through that, so apply Copernican Principle
//and assume we are halfway through
var error = $"Could not create queue {_subscription.ChannelName.Value} because {ex.Message} waiting 60s to retry";
s_logger.LogError(ex,
"Could not create queue {ChannelName} because {ErrorMessage} waiting 60s to retry",
_subscription.ChannelName.Value, ex.Message);
Thread.Sleep(TimeSpan.FromSeconds(30));
throw new ChannelFailureException(error, ex);
}
catch (AmazonSQSException ex)
{
var error = $"Could not create queue {_queueUrl} subscribed to topic {_subscription.RoutingKey.Value} in region {_awsConnection.Region.DisplayName} because {ex.Message}";
s_logger.LogError(ex,
"Could not create queue {URL} subscribed to topic {Topic} in region {Region} because {ErrorMessage}",
_queueUrl, _subscription.RoutingKey.Value, _awsConnection.Region.DisplayName, ex.Message);
throw new InvalidOperationException(error, ex);
}
catch (HttpErrorResponseException ex)
{
var error = $"Could not create queue {_queueUrl} subscribed to topic {_subscription.RoutingKey.Value} in region {_awsConnection.Region.DisplayName} because {ex.Message}";
s_logger.LogError(ex, "Could not create queue {URL} subscribed to topic {Topic} in region {Region} because {ErrorMessage}",
_queueUrl, _subscription.RoutingKey.Value, _awsConnection.Region.DisplayName, ex.Message);
throw new InvalidOperationException(error, ex);
}
}
public async Task CreateChannelsAsync(IEnumerable <string> channelNames, CancellationToken cancellationToken = default)
{
await Task.WhenAll(channelNames.Select(cn => _client.CreateQueueAsync(cn, cancellationToken))).ConfigureAwait(false);
}
public async Task CreateAsync(string name, CancellationToken cancellationToken)
{
var response = await _client.CreateQueueAsync(name, cancellationToken);
_logger.LogInformation("The request to create queue {0} return status code : {1}", name, response.HttpStatusCode);
}
