static async Task Main(string[] args)
{
AWSConfigs.LoggingConfig.LogResponses = ResponseLoggingOption.Always;
AWSConfigs.LoggingConfig.LogTo = LoggingOptions.Console;
var client = new AmazonSQSClient();
// Use any of these instead to see it work correctly
//var client = new AmazonSQSClient(new AmazonSQSConfig { ServiceURL = "http://sqs.us-east-1.amazonaws.com" });
//var client = new AmazonSQSClient(new AmazonSQSConfig { CacheHttpClient = false });
//var client = new AmazonSQSClient(new AmazonSQSConfig { HttpClientCacheSize = 3 });
var queueName1 = $"brandon-{Guid.NewGuid()}";
var queueName2 = $"brandon-{Guid.NewGuid()}";
var fifoQueueName = queueName2 + "-delay.fifo";
var tasks = new List <Task>
{
CreateQueue(client, queueName1, false),
CreateQueue(client, queueName2, true)
};
await Task.WhenAll(tasks);
queueUrls.TryGetValue(fifoQueueName, out var queueUrl);
var queueAttributes = await client.GetQueueAttributesAsync(queueUrl, new List <string> {
"DelaySeconds", "MessageRetentionPeriod", "RedrivePolicy"
});
if (queueAttributes.DelaySeconds < delaySeconds)
{
throw new Exception();
}
}
/// <summary>
/// Initializes the Amazon SQS client and then uses it to call the
/// GetQueueAttributesAsync method to retrieve the attributes for the
/// Amazon SQS queue.
/// </summary>
public static async Task Main()
{
// If the Amazon SQS message queue is not in the same AWS Region as your
// default user, you need to provide the AWS Region as a parameter to the
// client constructor.
var client = new AmazonSQSClient();
var queueUrl = "https://sqs.us-east-2.amazonaws.com/0123456789ab/New-Example-Queue";
var attrs = new List <string>()
{
"All"
};
var request = new GetQueueAttributesRequest
{
QueueUrl = queueUrl,
AttributeNames = attrs,
};
var response = await client.GetQueueAttributesAsync(request);
if (response.HttpStatusCode == System.Net.HttpStatusCode.OK)
{
DisplayAttributes(response);
}
}
async Task ConfigureLambdaWithQueueAsync(string queueName)
{
string queueArn = null;
AmazonSQSClient sqsClient = AwsFactory.CreateClient <AmazonSQSClient>();
GetQueueUrlRequest queueUrlReq = new GetQueueUrlRequest();
queueUrlReq.QueueName = queueName;
GetQueueUrlResponse getQueueUrlResp = await sqsClient.GetQueueUrlAsync(queueUrlReq);
GetQueueAttributesRequest queueAttribReq = new GetQueueAttributesRequest();
queueAttribReq.AttributeNames.Add(QueueAttributeName.QueueArn);
queueAttribReq.QueueUrl = getQueueUrlResp.QueueUrl;
var queueAttribResp = await sqsClient.GetQueueAttributesAsync(queueAttribReq);
queueArn = queueAttribResp.QueueARN;
AmazonLambdaClient lambdaClient = AwsFactory.CreateClient <AmazonLambdaClient>();
CreateEventSourceMappingRequest eventMappingReq = new CreateEventSourceMappingRequest();
eventMappingReq.FunctionName = "WebhookDispatcher";
eventMappingReq.BatchSize = 10;
eventMappingReq.Enabled = true;
eventMappingReq.EventSourceArn = queueArn;
await lambdaClient.CreateEventSourceMappingAsync(eventMappingReq);
}
static async Task Main(string[] args)
{
using var sqsClient = new AmazonSQSClient();
using var sqsClient2 = new AmazonSQSClient();
using var snsClient = new AmazonSimpleNotificationServiceClient();
using var snsClient2 = new AmazonSimpleNotificationServiceClient();
var queueUrl = await CreateQueue(sqsClient, QueueName);
var topic1Arn = await CreateTopic(snsClient, Topic1);
var topic2Arn = await CreateTopic(snsClient, Topic2);
async Task <string> Subscribe(AmazonSQSClient queueClient, AmazonSimpleNotificationServiceClient notificationClient, string topicArn)
{
try
{
await limitter.WaitAsync();
return(await notificationClient.SubscribeQueueAsync(topicArn, queueClient, queueUrl));
}
finally
{
limitter.Release();
}
}
var subscribeTask = new List <Task <string> > {
Subscribe(sqsClient, snsClient, topic1Arn), // feel free to experiment here with multiple clients, doesn't make the bug go away
Subscribe(sqsClient, snsClient, topic2Arn) // feel free to experiment here with multiple clients, doesn't make the bug go away
};
await Task.WhenAll(subscribeTask);
// give some time to settle although this is not necessary
await Task.Delay(5000);
var queueAttributes = await sqsClient.GetQueueAttributesAsync(queueUrl, new List <string> {
"All"
});
var policyAsPrettyJson = Policy.FromJson(queueAttributes.Attributes["Policy"]).ToJson(prettyPrint: true);
Console.WriteLine(policyAsPrettyJson);
var wasTopicOneContained = policyAsPrettyJson.Contains(topic1Arn);
var wasTopicTwoContained = policyAsPrettyJson.Contains(topic2Arn);
Console.WriteLine();
Console.WriteLine(wasTopicOneContained ? "Topic1 found in policy" : "Topic1 NOT found in policy!");
Console.WriteLine(wasTopicTwoContained ? "Topic2 found in policy" : "Topic2 NOT found in policy!");
Console.WriteLine("Hit enter to delete all created ressources");
Console.ReadLine();
Console.WriteLine("Deleting all ressources");
await Cleanup(sqsClient, snsClient, subscribeTask, topic1Arn, topic2Arn, queueUrl);
Console.WriteLine("Deleted all ressources");
Console.WriteLine("Hit enter to exit");
Console.ReadLine();
}
public override async Task <int> GetMessageCountAsync()
{
GetQueueAttributesResponse attrs = await _client.GetQueueAttributesAsync(_queueUrl, new List <string> {
"All"
}).ConfigureAwait(false);
return(attrs.ApproximateNumberOfMessages);
}
private void CreateDLQ(AmazonSQSClient sqsClient)
{
try
{
var request = new CreateQueueRequest(_subscription.RedrivePolicy.DeadlLetterQueueName.Value);
var createDeadLetterQueueResponse = sqsClient.CreateQueueAsync(request).GetAwaiter().GetResult();
var queueUrl = createDeadLetterQueueResponse.QueueUrl;
if (!string.IsNullOrEmpty(queueUrl))
{
//We need the ARN of the dead letter queue to configure the queue redrive policy, not the name
var attributesRequest = new GetQueueAttributesRequest
{
QueueUrl = queueUrl,
AttributeNames = new List <string> {
"QueueArn"
}
};
var attributesResponse = sqsClient.GetQueueAttributesAsync(attributesRequest).GetAwaiter().GetResult();
if (attributesResponse.HttpStatusCode != HttpStatusCode.OK)
{
throw new InvalidOperationException($"Could not find ARN of DLQ, status: {attributesResponse.HttpStatusCode}");
}
_dlqARN = attributesResponse.QueueARN;
}
else
{
throw new InvalidOperationException($"Could not find create DLQ, status: {createDeadLetterQueueResponse.HttpStatusCode}");
}
}
catch (QueueDeletedRecentlyException ex)
{
//QueueDeletedRecentlyException - wait 30 seconds then retry
//Although timeout is 60s, we could be partway through that, so apply Copernican Principle
//and assume we are halfway through
var error = $"Could not create queue {_subscription.ChannelName.Value} because {ex.Message} waiting 60s to retry";
s_logger.LogError(ex, error);
Thread.Sleep(TimeSpan.FromSeconds(30));
throw new ChannelFailureException(error, ex);
}
catch (AmazonSQSException ex)
{
var error = $"Could not create queue {_queueUrl} subscribed to topic {_subscription.RoutingKey.Value} in region {_awsConnection.Region.DisplayName} because {ex.Message}";
s_logger.LogError(ex, error);
throw new InvalidOperationException(error, ex);
}
catch (HttpErrorResponseException ex)
{
var error = $"Could not create queue {_queueUrl} subscribed to topic {_subscription.RoutingKey.Value} in region {_awsConnection.Region.DisplayName} because {ex.Message}";
s_logger.LogError(ex, error);
throw new InvalidOperationException(error, ex);
}
}
public async Task <State> GetStatusAsync(State state, ILambdaContext context)
{
var results = await SqsClient.GetQueueAttributesAsync(Config.LookupQueue, new List <string> {
"ApproximateNumberOfMessages"
});
state.QueueDepth = results.ApproximateNumberOfMessages;
state.WaitInSeconds = results.ApproximateNumberOfMessages < 10 ? 30 : 60;
return(state);
}
/// <inheritdoc/>
public async Task <long> GetMessageCountAsync(CancellationToken cancellationToken = default)
{
var request = new GetQueueAttributesRequest
{
QueueUrl = await GetQueueUrl().ConfigureAwait(false)
};
var response = await _client.GetQueueAttributesAsync(request, cancellationToken).ConfigureAwait(false);
return(response.ApproximateNumberOfMessages);
}
private async Task <IDictionary <string, string> > GetQueueAttributesAsync(string correlationId, List <string> attributes, string queueUrl)
{
var attributeMap = new Dictionary <string, string>();
try
{
var response = await _client.GetQueueAttributesAsync(new GetQueueAttributesRequest(queueUrl, attributes));
foreach (var attibute in response.Attributes)
{
attributeMap.Add(attibute.Key, attibute.Value);
}
}
catch (InvalidAttributeNameException)
{
// Ignore invalid name exception
}
return(attributeMap);
}
public async Task <int> MessageCount(string queueName)
{
var getQueueAttributesRequest = new GetQueueAttributesRequest
{
QueueUrl = QueueUrlFactory(queueName),
AttributeNames = new List <string> {
"ApproximateNumberOfMessages", "ApproximateNumberOfMessagesNotVisible"
}
};
var getQueueAttributesResponse = await _sqsClient.GetQueueAttributesAsync(getQueueAttributesRequest);
// for FIFO queues this number is, in fact, exact
return(getQueueAttributesResponse.ApproximateNumberOfMessages + getQueueAttributesResponse.ApproximateNumberOfMessagesNotVisible);
}
public async Task <int> CountMessages(string url = null)
{
if (url == null)
{
url = awsSQSOptions.Url;
}
var getQueueAttributesResponse = await amazonSQSClient.GetQueueAttributesAsync(new GetQueueAttributesRequest()
{
QueueUrl = url,
AttributeNames = { "ApproximateNumberOfMessages" }
});
return(getQueueAttributesResponse.ApproximateNumberOfMessages);
}
private static int NumbersOfMessagesInQueue(string queueUrl, AmazonSQSClient sqsClient)
{
GetQueueAttributesRequest attReq = new GetQueueAttributesRequest();
attReq.QueueUrl = queueUrl;
attReq.AttributeNames.Add("ApproximateNumberOfMessages");
GetQueueAttributesResponse response =
Task.Run(async() => await sqsClient.GetQueueAttributesAsync(attReq)).Result;
var retval = response.ApproximateNumberOfMessages;
return(retval);
}
public static async Task<int> QueueLengthByQueueUrl(string queueUrl)
{
using (var client = new AmazonSQSClient(ConfigManager.ConfigSettings.AccessKey, ConfigManager.ConfigSettings.Secret, Amazon.RegionEndpoint.USWest2))
{
var request = new GetQueueAttributesRequest
{
QueueUrl = queueUrl,
AttributeNames = new List<string> { "ApproximateNumberOfMessages" }
};
var response = await client.GetQueueAttributesAsync(request);
return response.ApproximateNumberOfMessages;
}
}
public bool QueueEmpty()
{
using (var sqsClient = new AmazonSQSClient())
{
var attr = sqsClient.GetQueueAttributesAsync(new GetQueueAttributesRequest
{
AttributeNames = new List <string> {
"ApproximateNumberOfMessages"
},
QueueUrl = Environment.GetEnvironmentVariable("Queue")
});
Console.WriteLine($"Queue count: {0}", attr.Result.ApproximateNumberOfMessages);
return(attr.Result.ApproximateNumberOfMessages == 0);
}
}
private string GetQueueARNForChannel(AmazonSQSClient sqsClient)
{
var result = sqsClient.GetQueueAttributesAsync(
new GetQueueAttributesRequest {
QueueUrl = _queueUrl, AttributeNames = new List <string> {
"QueueArn"
}
}
).GetAwaiter().GetResult();
if (result.HttpStatusCode == HttpStatusCode.OK)
{
return(result.QueueARN);
}
return(null);
}
public void CreateQueue(AmazonSQSClient client)
{
CreateQueueRequest createDlqRequest = new CreateQueueRequest
{
QueueName = "ArmutLocalStack-Test-DLQ.fifo",
Attributes = new Dictionary <string, string>
{
{
"FifoQueue", "true"
},
}
};
CreateQueueResponse createDlqResult =
client.CreateQueueAsync(createDlqRequest).GetAwaiter().GetResult();
var attributes = client.GetQueueAttributesAsync(new GetQueueAttributesRequest
{
QueueUrl = createDlqResult.QueueUrl,
AttributeNames = new List <string> {
"QueueArn"
}
}).GetAwaiter().GetResult();
var redrivePolicy = new
{
maxReceiveCount = "1",
deadLetterTargetArn = attributes.Attributes["QueueArn"]
};
CreateQueueRequest createQueueRequest = new CreateQueueRequest
{
QueueName = "ArmutLocalStack-Test.fifo",
Attributes = new Dictionary <string, string>
{
{
"FifoQueue", "true"
},
{
"RedrivePolicy", JsonSerializer.Serialize(redrivePolicy)
},
}
};
CreateQueueResponse createQueueResult =
client.CreateQueueAsync(createQueueRequest).GetAwaiter().GetResult();
}
private List <QueueMessage> GetMessageRepository(string queueURL)
{
List <QueueMessage> listOfMessage = new List <QueueMessage>();
int countOfMessages = 0;
var getNumberOfMessagesRequest = sqsClient.GetQueueAttributesAsync(
new GetQueueAttributesRequest
{
QueueUrl = queueURL,
AttributeNames = new List <string>()
{
"ApproximateNumberOfMessages"
}
}
);
countOfMessages = getNumberOfMessagesRequest.Result.ApproximateNumberOfMessages;
for (int i = 0; i < countOfMessages; i++)
{
ReceiveMessageRequest receiveMessageRequest = new ReceiveMessageRequest();
receiveMessageRequest.QueueUrl = queueURL;
receiveMessageRequest.MaxNumberOfMessages = 10;
receiveMessageRequest.AttributeNames = new List <string>()
{
"ApproximateReceiveCount"
};
var receiveMessageResponse = sqsClient.ReceiveMessageAsync(receiveMessageRequest);
foreach (var message in receiveMessageResponse.Result.Messages)
{
string ReceiveCount_;
message.Attributes.TryGetValue("ApproximateReceiveCount", out ReceiveCount_);
listOfMessage.Add(
new QueueMessage
{
Body = message.Body,
Id = message.MessageId,
ReceiveCount = Int32.Parse(ReceiveCount_),
Size = message.Body.Length
});
}
}
return(listOfMessage);
}
private async Task SubscribeTopicToQueue()
{
var currentSubscriptions = (await _snsClient.ListSubscriptionsByTopicAsync(_topicArn)).Subscriptions;
if (currentSubscriptions.Any())
{
var queueArn = (await _sqsClient.GetQueueAttributesAsync(_queueUrl, new List <string> {
"QueueArn"
})).QueueARN;
var existingSubscription = currentSubscriptions.FirstOrDefault(x => x.Endpoint == queueArn);
if (existingSubscription != null)
{
return;
}
}
await _snsClient.SubscribeQueueAsync(_topicArn, _sqsClient, _queueUrl);
}
public async System.Threading.Tasks.Task <GetQueueAttributesResponse> GetQueueAttributes(string queueUrl)
{
GetQueueAttributesResponse queueAttributesResponse = new GetQueueAttributesResponse();
using (AmazonSQSClient sqsClient = new AmazonSQSClient(credentials, Amazon.RegionEndpoint.USEast2))
{
GetQueueAttributesRequest request = new GetQueueAttributesRequest
{
QueueUrl = queueUrl,
AttributeNames = new List <string>()
{
"All"
}
};
queueAttributesResponse = await sqsClient.GetQueueAttributesAsync(request);
}
return(queueAttributesResponse);
}
/// <summary>
/// Gets information regarding the number of messages on a queue.
/// </summary>
/// <param name="queueName">The name of the queue</param>
/// <returns></returns>
public async Task <NumberOfMessagesResponse> GetNumberOfMessagesOnQueue(string queueName)
{
var queueUrl = await GetQueueUrlAsync(queueName);
var response = await _client.GetQueueAttributesAsync(new GetQueueAttributesRequest(queueUrl, new List <string>
{
"ApproximateNumberOfMessages",
"ApproximateNumberOfMessagesNotVisible",
"ApproximateNumberOfMessagesDelayed"
}));
return(new NumberOfMessagesResponse
{
QueueName = queueName,
ApproximateNumberOfMessages = response.ApproximateNumberOfMessages,
ApproximateNumberOfMessagesNotVisible = response.ApproximateNumberOfMessagesNotVisible,
ApproximateNumberOfMessagesDelayed = response.ApproximateNumberOfMessagesDelayed
});
}
public async Task <ISubscription> SubscribeAsync(String topic)
{
var amazonTopic = await amazonSnsClient.CreateTopicAsync(new CreateTopicRequest()
{
Name = topic
});
var queue = await amazonSqsClient.CreateQueueAsync(new CreateQueueRequest()
{
QueueName = Guid.NewGuid().ToString()
});
var queueAttributes = await amazonSqsClient.GetQueueAttributesAsync(new GetQueueAttributesRequest()
{
AttributeNames = new List <String>(new String[] { "QueueArn" }),
QueueUrl = queue.QueueUrl
});
var policy = new Policy()
.WithStatements(
new Statement(Statement.StatementEffect.Allow)
.WithPrincipals(Principal.AllUsers)
.WithConditions(ConditionFactory.NewSourceArnCondition(amazonTopic.TopicArn))
.WithResources(new Resource(queueAttributes.QueueARN))
.WithActionIdentifiers(SQSActionIdentifiers.SendMessage));
await amazonSqsClient.SetQueueAttributesAsync(queue.QueueUrl, new Dictionary <String, String>()
{
["Policy"] = policy.ToJson()
});
await amazonSnsClient.SubscribeAsync(new SubscribeRequest()
{
Endpoint = queueAttributes.QueueARN,
Protocol = "sqs",
TopicArn = amazonTopic.TopicArn
});
var subscription = amazonSubscriptionFactory.Create(queue.QueueUrl);
return(subscription);
}
static async Task Main(string[] args)
{
string queueUrl = $"https://sqs.<Region e.g. us-east-2>.amazonaws.com/<Account number>/{args[0]}";
const string accessKey = "<Access Key>";
const string secretKey = "<Secret Key>";
var attributes = new List <string> {
"ApproximateNumberOfMessages", "ApproximateNumberOfMessagesNotVisible"
};
IAmazonSQS sqsClient = new AmazonSQSClient(accessKey, secretKey, RegionEndpoint.USEast2);
while (true)
{
var response = await sqsClient.GetQueueAttributesAsync(queueUrl, attributes);
Console.Clear();
Console.WriteLine($"Visible messages: {response.ApproximateNumberOfMessages} msgs");
Console.WriteLine($"Processing messages: {response.ApproximateNumberOfMessagesNotVisible} msgs");
Thread.Sleep(1000);
}
}
public async Task <long> GetMessageCountAsync(string channelName, CancellationToken cancellationToken = default)
{
if (channelName is null)
{
throw new ArgumentNullException(nameof(channelName));
}
try
{
GetQueueAttributesResponse attributes =
await _client.GetQueueAttributesAsync(GetQueueUri(channelName), new List <string> {
"All"
}, cancellationToken).ConfigureAwait(false);
return(attributes.ApproximateNumberOfMessages);
}
catch (AmazonSQSException ex) when(ex.ErrorCode == "AWS.SimpleQueueService.NonExistentQueue")
{
return(0);
}
}
async Task <CreateQueueResponse> CreateQueueAsync(string queueName, string SubscriptionId)
{
CreateQueueRequest deadLetterRequest = new CreateQueueRequest(string.Concat(queueName, "-deadletter"));
deadLetterRequest.Attributes = new Dictionary <string, string>();
deadLetterRequest.Attributes.Add(QueueAttributeName.ReceiveMessageWaitTimeSeconds, "20");
deadLetterRequest.Attributes.Add(QueueAttributeName.MessageRetentionPeriod, "864000");
string deadLetterArn = null;
AmazonSQSClient sqsClient = AwsFactory.CreateClient <AmazonSQSClient>();
var createResponse = await sqsClient.CreateQueueAsync(deadLetterRequest);
GetQueueAttributesRequest queueReq = new GetQueueAttributesRequest();
queueReq.QueueUrl = createResponse.QueueUrl;
queueReq.AttributeNames.Add(QueueAttributeName.All);
var queueAttribs = await sqsClient.GetQueueAttributesAsync(queueReq);
deadLetterArn = queueAttribs.QueueARN;
string redrivePolicy = $"{{\"deadLetterTargetArn\":\"{deadLetterArn}\",\"maxReceiveCount\":5}}";
CreateQueueRequest createQueueRequest = new CreateQueueRequest();
createQueueRequest.QueueName = queueName;
createQueueRequest.Attributes = new Dictionary <string, string>();
createQueueRequest.Attributes.Add(QueueAttributeName.RedrivePolicy, redrivePolicy);
createQueueRequest.Attributes.Add(QueueAttributeName.ReceiveMessageWaitTimeSeconds, "20");
//createQueueRequest.Attributes.Add("trigger-id", SubscriptionId);
CreateQueueResponse queueResponse = await sqsClient.CreateQueueAsync(createQueueRequest);
return(queueResponse);
}
public async void TestQueueSubscription()
{
var topicArn = await CreateTopic();
var queueUrl = await CreateQueue();
var subscriptionArn = await SubscribeQueue(topicArn, queueUrl);
var publishRequest = GetPublishRequest(topicArn);
List <Message> messages = await PublishToSNSAndReceiveMessages(publishRequest, topicArn, queueUrl);
Assert.Equal(1, messages.Count);
var message = messages[0];
string bodyJson = GetBodyJson(message);
var json = ThirdParty.Json.LitJson.JsonMapper.ToObject(bodyJson);
var messageText = json["Message"];
var messageSubject = json["Subject"];
Assert.Equal(publishRequest.Message, messageText.ToString());
Assert.Equal(publishRequest.Subject, messageSubject.ToString());
var messageAttributes = json["MessageAttributes"];
Assert.Equal(publishRequest.MessageAttributes.Count, messageAttributes.Count);
foreach (var ma in publishRequest.MessageAttributes)
{
var name = ma.Key;
var value = ma.Value;
Assert.True(messageAttributes.PropertyNames.Contains(name, StringComparer.Ordinal));
var jsonAttribute = messageAttributes[name];
var jsonType = jsonAttribute["Type"].ToString();
var jsonValue = jsonAttribute["Value"].ToString();
Assert.NotNull(jsonType);
Assert.NotNull(jsonValue);
Assert.Equal(value.DataType, jsonType);
Assert.Equal(value.DataType != "Binary"
? value.StringValue
: Convert.ToBase64String(value.BinaryValue.ToArray()), jsonValue);
}
await sqsClient.DeleteMessageAsync(new DeleteMessageRequest
{
QueueUrl = queueUrl,
ReceiptHandle = messages[0].ReceiptHandle
});
// This will unsubscribe but leave the policy in place.
await Client.UnsubscribeAsync(new UnsubscribeRequest
{
SubscriptionArn = subscriptionArn
});
// Subscribe again to see if this affects the policy.
await Client.SubscribeQueueAsync(topicArn, sqsClient as ICoreAmazonSQS, queueUrl);
await Client.PublishAsync(new PublishRequest
{
TopicArn = topicArn,
Message = "Test Message again"
});
messages = (await sqsClient.ReceiveMessageAsync(new ReceiveMessageRequest
{
QueueUrl = queueUrl,
WaitTimeSeconds = 20
})).Messages;
Assert.Equal(1, messages.Count);
var response = WaitUtils.WaitForComplete(
() => {
return(sqsClient.GetQueueAttributesAsync(new GetQueueAttributesRequest
{
AttributeNames = new List <string> {
"All"
},
QueueUrl = queueUrl
}).Result);
},
(r) =>
{
return(!string.IsNullOrEmpty(r.Policy));
});
var policy = Policy.FromJson(response.Policy);
Assert.Equal(1, policy.Statements.Count);
}
private static async Task <Topic> SetupTopicAndSubscriptionsAsync(string topicFileName, string outputDirectory, string regionSystemName, string archiveId = null, string filename = null)
{
var topic = new Topic
{
TopicFileName = topicFileName,
OutputDirectory = outputDirectory,
ArchiveId = archiveId,
FileName = filename,
DateRequested = DateTime.Now
};
long ticks = DateTime.Now.Ticks;
var settings = GetSettingsAsync().Result;
#region Setup SNS topic
var snsClient = new AmazonSimpleNotificationServiceClient(
settings.AWSAccessKeyID,
settings.AWSSecretAccessKey,
RegionEndpoint.GetBySystemName(regionSystemName));
var sqsClient = new AmazonSQSClient(
settings.AWSAccessKeyID,
settings.AWSSecretAccessKey,
RegionEndpoint.GetBySystemName(regionSystemName));
var topicArn = snsClient.CreateTopicAsync(new CreateTopicRequest {
Name = "GlacierDownload-" + ticks
}).Result.TopicArn;
//Debug.WriteLine($"topicArn: {topicArn}");
topic.TopicARN = topicArn;
#endregion
#region Setup SQS queue
var createQueueRequest = new CreateQueueRequest {
QueueName = "GlacierDownload-" + ticks
};
var createQueueResponse = sqsClient.CreateQueueAsync(createQueueRequest).Result;
var queueUrl = createQueueResponse.QueueUrl;
//Debug.WriteLine($"QueueURL: {queueUrl}");
topic.QueueUrl = queueUrl;
var getQueueAttributesRequest = new GetQueueAttributesRequest
{
AttributeNames = new List <string> {
"QueueArn"
},
QueueUrl = queueUrl
};
var response = await sqsClient.GetQueueAttributesAsync(getQueueAttributesRequest);
var queueArn = response.QueueARN;
Debug.WriteLine($"QueueArn: {queueArn}");
topic.QueueARN = queueArn;
#endregion
// Setup the Amazon SNS topic to publish to the SQS queue.
// TODO SMS subscription
await snsClient.SubscribeAsync(new SubscribeRequest()
{
Protocol = "sqs",
Endpoint = queueArn,
TopicArn = topicArn
});
// Add the policy to the queue so SNS can send messages to the queue.
var policy = SQS_POLICY.Replace("{TopicArn}", topicArn).Replace("{QuernArn}", queueArn);
await sqsClient.SetQueueAttributesAsync(new SetQueueAttributesRequest
{
QueueUrl = queueUrl,
Attributes = new Dictionary <string, string>
{
{ QueueAttributeName.Policy, policy }
}
});
return(topic);
}
public async Task FixIssueOfDLQBeingCleared()
{
var sqsClient = new AmazonSQSClient(RegionEndpoint.USEast2);
var queueUrl = (await sqsClient.CreateQueueAsync("lambda-test-" + DateTime.Now.Ticks)).QueueUrl;
var queueArn = (await sqsClient.GetQueueAttributesAsync(queueUrl, new List <string> {
"QueueArn"
})).QueueARN;
try
{
var assembly = this.GetType().GetTypeInfo().Assembly;
var fullPath = Path.GetFullPath(Path.GetDirectoryName(assembly.Location) + "../../../../../../testapps/TestFunction");
var initialDeployCommand = new DeployFunctionCommand(new TestToolLogger(_testOutputHelper), fullPath, new string[0]);
initialDeployCommand.FunctionName = "test-function-" + DateTime.Now.Ticks;
initialDeployCommand.Handler = "TestFunction::TestFunction.Function::ToUpper";
initialDeployCommand.Timeout = 10;
initialDeployCommand.MemorySize = 512;
initialDeployCommand.Role = TestHelper.GetTestRoleArn();
initialDeployCommand.Configuration = "Release";
initialDeployCommand.TargetFramework = "netcoreapp1.0";
initialDeployCommand.Runtime = "dotnetcore1.0";
initialDeployCommand.DeadLetterTargetArn = queueArn;
initialDeployCommand.DisableInteractive = true;
var created = await initialDeployCommand.ExecuteAsync();
try
{
Assert.True(created);
var funcConfig = await initialDeployCommand.LambdaClient.GetFunctionConfigurationAsync(initialDeployCommand.FunctionName);
Assert.Equal(queueArn, funcConfig.DeadLetterConfig?.TargetArn);
var redeployCommand = new DeployFunctionCommand(new TestToolLogger(_testOutputHelper), fullPath, new string[0]);
redeployCommand.FunctionName = initialDeployCommand.FunctionName;
redeployCommand.Configuration = "Release";
redeployCommand.TargetFramework = "netcoreapp1.0";
redeployCommand.Runtime = "dotnetcore1.0";
redeployCommand.DisableInteractive = true;
var redeployed = await redeployCommand.ExecuteAsync();
Assert.True(redeployed);
funcConfig = await initialDeployCommand.LambdaClient.GetFunctionConfigurationAsync(initialDeployCommand.FunctionName);
Assert.Equal(queueArn, funcConfig.DeadLetterConfig?.TargetArn);
redeployCommand = new DeployFunctionCommand(new TestToolLogger(_testOutputHelper), fullPath, new string[0]);
redeployCommand.FunctionName = initialDeployCommand.FunctionName;
redeployCommand.Configuration = "Release";
redeployCommand.TargetFramework = "netcoreapp1.0";
redeployCommand.Runtime = "dotnetcore1.0";
redeployCommand.DeadLetterTargetArn = "";
redeployCommand.DisableInteractive = true;
redeployed = await redeployCommand.ExecuteAsync();
Assert.True(redeployed);
funcConfig = await initialDeployCommand.LambdaClient.GetFunctionConfigurationAsync(initialDeployCommand.FunctionName);
Assert.Null(funcConfig.DeadLetterConfig?.TargetArn);
}
finally
{
if (created)
{
await initialDeployCommand.LambdaClient.DeleteFunctionAsync(initialDeployCommand.FunctionName);
}
}
}
finally
{
await sqsClient.DeleteQueueAsync(queueUrl);
}
}
public void MonitorEmail(string emailaddr)
{
if (Exists(emailaddr) && !monitors.ContainsKey(emailaddr))
{
FastEmailMailbox mbx = Mailboxes[emailaddr];
FastEmailMonitor monitor = new FastEmailMonitor();
monitor.mbx = mbx;
monitor.sqsqueue_name = mbx.S3Bucket + "-" + Process.GetCurrentProcess().Id + "-" + System.Environment.MachineName;
create_sqsclient();
try
{
// create the queue
var sqscreateresponse = sqsclient.CreateQueueAsync(new CreateQueueRequest
{
QueueName = monitor.sqsqueue_name
});
CreateQueueResponse sqsresult = sqscreateresponse.Result;
monitor.sqsqueue_url = sqsresult.QueueUrl;
}
catch (AmazonSQSException e)
{
Console.WriteLine("Exception while creating SQS Queue for {0}: {1}", emailaddr, e.Message);
}
// get the queue arn
try
{
List <string> attr = new List <string>()
{
"QueueArn"
};
var sqsattrresponse = sqsclient.GetQueueAttributesAsync(new GetQueueAttributesRequest
{
QueueUrl = monitor.sqsqueue_url,
AttributeNames = attr
});
GetQueueAttributesResponse sqsresponse = sqsattrresponse.Result;
monitor.sqsqueue_arn = sqsresponse.QueueARN;
}
catch (AmazonSQSException e)
{
Console.WriteLine("Exception while getting QueueARN SQS Queue for {0}: {1}", emailaddr, e.Message);
}
// add permission
string perm = @"{
""Version"":""2012-10-17"",
""Statement"":[
{
""Sid"":""Policy-"
+ monitor.mbx.TopicArn;
perm += @""",
""Effect"":""Allow"",
""Principal"":""*"",
""Action"":""sqs:SendMessage"",
""Resource"":"""
+ monitor.sqsqueue_arn;
perm += @""",
""Condition"":{
""ArnEquals"":{
""aws:SourceArn"":"""
+ monitor.mbx.TopicArn;
perm += @"""
}
}
}
]
}";
var policy = new Dictionary <string, string>();
policy.Add("Policy", perm);
try
{
var qsattrrequest = sqsclient.SetQueueAttributesAsync(new SetQueueAttributesRequest
{
QueueUrl = monitor.sqsqueue_url,
Attributes = policy
});
var qsattrresponse = qsattrrequest.Result;
}
catch (AmazonSQSException e)
{
Console.WriteLine("Exception while adding permission policy to queue for {0}: {1}", emailaddr, e.Message);
}
create_snsclient();
try
{
var snsresponse = snsclient.SubscribeAsync(new SubscribeRequest
{
Protocol = "sqs",
Endpoint = monitor.sqsqueue_arn,
TopicArn = monitor.mbx.TopicArn
});
var subresult = snsresponse.Result;
monitor.subscription_arn = subresult.SubscriptionArn;
}
catch (AmazonSimpleNotificationServiceException e)
{
Console.WriteLine("Exception while subscribing to queue for {0}: {1}", emailaddr, e.Message);
}
monitors.Add(emailaddr, monitor);
}
}
public async void TestQueueSubscription()
{
// create new topic
var topicName = UtilityMethods.GenerateName("TestQueueSubscription");
var createTopicRequest = new CreateTopicRequest
{
Name = topicName
};
var createTopicResult = await Client.CreateTopicAsync(createTopicRequest);
var topicArn = createTopicResult.TopicArn;
_topicArns.Add(topicArn);
var queueName = UtilityMethods.GenerateName("TestQueueSubscription");
var queueUrl = (await sqsClient.CreateQueueAsync(new CreateQueueRequest
{
QueueName = queueName
})).QueueUrl;
_queueUrl.Add(queueUrl);
ICoreAmazonSQS coreSqs = sqsClient as ICoreAmazonSQS;
var subscriptionARN = await Client.SubscribeQueueAsync(topicArn, coreSqs, queueUrl);
// Sleep to wait for the subscribe to complete.
Thread.Sleep(TimeSpan.FromSeconds(5));
var publishRequest = new PublishRequest
{
TopicArn = topicArn,
Subject = "Test Subject",
Message = "Test Message",
MessageAttributes = new Dictionary <string, SNSMessageAttributeValue>
{
{ "Color", new SNSMessageAttributeValue {
StringValue = "Red", DataType = "String"
} },
{ "Binary", new SNSMessageAttributeValue {
DataType = "Binary", BinaryValue = new MemoryStream(Encoding.UTF8.GetBytes("Yes please"))
} },
{ "Prime", new SNSMessageAttributeValue {
StringValue = "31", DataType = "Number"
} },
}
};
await Client.PublishAsync(publishRequest);
var messages = (await sqsClient.ReceiveMessageAsync(new ReceiveMessageRequest
{
QueueUrl = queueUrl,
WaitTimeSeconds = 20
})).Messages;
Assert.Equal(1, messages.Count);
var message = messages[0];
string bodyJson;
// Handle some accounts returning message body as base 64 encoded.
if (message.Body.Trim()[0] == '{')
{
bodyJson = message.Body;
}
else
{
bodyJson = Encoding.UTF8.GetString(Convert.FromBase64String(message.Body));
}
var json = ThirdParty.Json.LitJson.JsonMapper.ToObject(bodyJson);
var messageText = json["Message"];
var messageSubject = json["Subject"];
Assert.Equal(publishRequest.Message, messageText.ToString());
Assert.Equal(publishRequest.Subject, messageSubject.ToString());
var messageAttributes = json["MessageAttributes"];
Assert.Equal(publishRequest.MessageAttributes.Count, messageAttributes.Count);
foreach (var ma in publishRequest.MessageAttributes)
{
var name = ma.Key;
var value = ma.Value;
Assert.True(messageAttributes.PropertyNames.Contains(name, StringComparer.Ordinal));
var jsonAttribute = messageAttributes[name];
var jsonType = jsonAttribute["Type"].ToString();
var jsonValue = jsonAttribute["Value"].ToString();
Assert.NotNull(jsonType);
Assert.NotNull(jsonValue);
Assert.Equal(value.DataType, jsonType);
Assert.Equal(value.DataType != "Binary"
? value.StringValue
: Convert.ToBase64String(value.BinaryValue.ToArray()), jsonValue);
}
await sqsClient.DeleteMessageAsync(new DeleteMessageRequest
{
QueueUrl = queueUrl,
ReceiptHandle = messages[0].ReceiptHandle
});
// This will unsubscribe but leave the policy in place.
await Client.UnsubscribeAsync(new UnsubscribeRequest
{
SubscriptionArn = subscriptionARN
});
// Subscribe again to see if this affects the policy.
await Client.SubscribeQueueAsync(topicArn, coreSqs, queueUrl);
await Client.PublishAsync(new PublishRequest
{
TopicArn = topicArn,
Message = "Test Message again"
});
messages = (await sqsClient.ReceiveMessageAsync(new ReceiveMessageRequest
{
QueueUrl = queueUrl,
WaitTimeSeconds = 20
})).Messages;
Assert.Equal(1, messages.Count);
var response = WaitUtils.WaitForComplete(
() => {
return(sqsClient.GetQueueAttributesAsync(new GetQueueAttributesRequest
{
AttributeNames = new List <string> {
"All"
},
QueueUrl = queueUrl
}).Result);
},
(r) =>
{
return(!string.IsNullOrEmpty(r.Policy));
});
var policy = Policy.FromJson(response.Policy);
Assert.Equal(1, policy.Statements.Count);
}