public void Parsing_s3_bucket_works()
{
var extensions = new TransportExtensions <SqsTransport>(new SettingsHolder());
var result = extensions.S3("myTestBucket", "blah\blah");
Assert.AreEqual("myTestBucket", result.GetSettings().Get("NServiceBus.AmazonSQS.S3BucketForLargeMessages"));
}
public void S3_server_side_customer_settings_provided_key_not_empty()
{
var extensions = new TransportExtensions <SqsTransport>(new SettingsHolder());
var s3Settings = extensions.S3("myTestBucket", "blah\blah");
Assert.Throws <ArgumentException>(() => s3Settings.ServerSideCustomerEncryption(ServerSideEncryptionCustomerMethod.AES256, ""));
}
public void S3_customer_settings_cannot_be_used_with_server_side_settings()
{
var extensions = new TransportExtensions <SqsTransport>(new SettingsHolder());
var s3Settings = extensions.S3("myTestBucket", "blah\blah");
s3Settings.ServerSideCustomerEncryption(ServerSideEncryptionCustomerMethod.AES256, "SomeKey");
Assert.Throws <InvalidOperationException>(() => s3Settings.ServerSideEncryption(ServerSideEncryptionMethod.AES256, "SomeKey"));
}
public void Parsing_s3_server_side_settings_works()
{
var extensions = new TransportExtensions <SqsTransport>(new SettingsHolder());
var s3Settings = extensions.S3("myTestBucket", "blah\blah");
s3Settings.ServerSideEncryption(ServerSideEncryptionMethod.AES256, "SomeKey");
Assert.AreEqual(ServerSideEncryptionMethod.AES256, s3Settings.GetSettings().Get("NServiceBus.AmazonSQS.ServerSideEncryptionMethod"));
Assert.AreEqual("SomeKey", s3Settings.GetSettings().Get("NServiceBus.AmazonSQS.ServerSideEncryptionKeyManagementServiceKeyId"));
}
public static void ConfigureSqsTransport(this TransportExtensions <SqsTransport> transportConfiguration, string queueNamePrefix = null)
{
transportConfiguration.ClientFactory(ClientFactory.CreateSqsClient);
if (queueNamePrefix != null)
{
transportConfiguration.QueueNamePrefix(queueNamePrefix);
}
var s3BucketName = EnvironmentHelper.GetEnvironmentVariable(S3BucketEnvironmentVariableName);
if (!string.IsNullOrEmpty(S3BucketName))
{
transportConfiguration.S3(S3BucketName, "test");
}
}
public static TransportExtensions <SqsTransport> ConfigureSqsTransport(this TransportExtensions <SqsTransport> transportConfiguration, string queueNamePrefix)
{
transportConfiguration
.ClientFactory(CreateSQSClient)
.QueueNamePrefix(queueNamePrefix)
.PreTruncateQueueNamesForAcceptanceTests();
var s3BucketName = EnvironmentHelper.GetEnvironmentVariable(S3BucketEnvironmentVariableName);
if (!string.IsNullOrEmpty(s3BucketName))
{
var s3Configuration = transportConfiguration.S3(s3BucketName, "test");
s3Configuration.ClientFactory(CreateS3Client);
}
return(transportConfiguration);
}
public async Task Should_upload_large_isolated_operations_request_to_s3()
{
var settings = new SettingsHolder();
var transportExtensions = new TransportExtensions <SqsTransport>(settings);
transportExtensions.S3("someBucket", "somePrefix");
var mockS3Client = new MockS3Client();
var mockSqsClient = new MockSqsClient();
var transportConfiguration = new TransportConfiguration(settings);
var dispatcher = new MessageDispatcher(transportConfiguration, mockS3Client, mockSqsClient, null, new QueueCache(mockSqsClient, transportConfiguration), null);
var transportOperations = new TransportOperations(
new TransportOperation(
new OutgoingMessage(Guid.NewGuid().ToString(), new Dictionary <string, string>(), Encoding.Default.GetBytes(new string('x', 256 * 1024))),
new UnicastAddressTag("address1"),
DispatchConsistency.Isolated),
new TransportOperation(
new OutgoingMessage(Guid.NewGuid().ToString(), new Dictionary <string, string>(), Encoding.Default.GetBytes(new string('x', 256 * 1024))),
new UnicastAddressTag("address2"),
DispatchConsistency.Isolated),
new TransportOperation( /* Crazy long message id will cause the message to go over limits because attributes count as well */
new OutgoingMessage(new string('x', 256 * 1024), new Dictionary <string, string>(), Encoding.Default.GetBytes("{}")),
new UnicastAddressTag("address2"),
DispatchConsistency.Isolated));
var transportTransaction = new TransportTransaction();
var context = new ContextBag();
await dispatcher.Dispatch(transportOperations, transportTransaction, context);
Assert.AreEqual(3, mockSqsClient.RequestsSent.Count);
Assert.AreEqual(3, mockS3Client.PutObjectRequestsSent.Count);
var firstUpload = mockS3Client.PutObjectRequestsSent.ElementAt(0);
var secondUpload = mockS3Client.PutObjectRequestsSent.ElementAt(1);
var thirdUpload = mockS3Client.PutObjectRequestsSent.ElementAt(2);
Assert.AreEqual("someBucket", firstUpload.BucketName);
Assert.AreEqual("someBucket", secondUpload.BucketName);
Assert.AreEqual("someBucket", thirdUpload.BucketName);
StringAssert.Contains($@"""Body"":"""",""S3BodyKey"":""{firstUpload.Key}", mockSqsClient.RequestsSent.ElementAt(0).MessageBody);
StringAssert.Contains($@"""Body"":"""",""S3BodyKey"":""{secondUpload.Key}", mockSqsClient.RequestsSent.ElementAt(1).MessageBody);
StringAssert.Contains($@"""Body"":"""",""S3BodyKey"":""{thirdUpload.Key}", mockSqsClient.RequestsSent.ElementAt(2).MessageBody);
}
public static void ConfigureSqsTransport(this TransportExtensions <SqsTransport> transportConfiguration, string queueNamePrefix = null)
{
transportConfiguration
.ClientFactory(() => new AmazonSQSClient(new AmazonSQSConfig {
RegionEndpoint = RegionEndpoint.APSoutheast2
}))
.QueueNamePrefix(queueNamePrefix);
var s3BucketName = EnvironmentHelper.GetEnvironmentVariable(S3BucketEnvironmentVariableName);
if (!string.IsNullOrEmpty(S3BucketName))
{
transportConfiguration.S3(S3BucketName, "test");
}
var nativeDeferralRaw = EnvironmentHelper.GetEnvironmentVariable(NativeDeferralEnvironmentVariableName);
var validValue = bool.TryParse(nativeDeferralRaw, out var nativeDeferral);
if (validValue && nativeDeferral)
{
transportConfiguration.NativeDeferral();
}
}
static void ConfigureTransport(TransportExtensions <SqsTransport> transport, TransportSettings transportSettings)
{
var builder = new DbConnectionStringBuilder {
ConnectionString = transportSettings.ConnectionString
};
var alwaysLoadFromEnvironmentVariable = false;
if (builder.ContainsKey("AccessKeyId") || builder.ContainsKey("SecretAccessKey"))
{
PromoteEnvironmentVariableFromConnectionString(builder, "AccessKeyId", "AWS_ACCESS_KEY_ID");
PromoteEnvironmentVariableFromConnectionString(builder, "SecretAccessKey", "AWS_SECRET_ACCESS_KEY");
// if the user provided the access key and secret access key they should always be loaded from environment credentials
alwaysLoadFromEnvironmentVariable = true;
transport.ClientFactory(() => new AmazonSQSClient(new EnvironmentVariablesAWSCredentials()));
}
else
{
//See https://docs.aws.amazon.com/sdk-for-net/v3/developer-guide/net-dg-config-creds.html#creds-assign
log.Info("BasicAWSCredentials have not been supplied in the connection string. Attempting to use existing environment or IAM role credentials for SQS Client.");
}
var region = PromoteEnvironmentVariableFromConnectionString(builder, "Region", "AWS_REGION");
var awsRegion = RegionEndpoint.EnumerableAllRegions
.SingleOrDefault(x => x.SystemName == region);
if (awsRegion == null)
{
throw new ArgumentException($"Unknown region: \"{region}\"");
}
if (builder.TryGetValue("QueueNamePrefix", out var queueNamePrefix))
{
var queueNamePrefixAsString = (string)queueNamePrefix;
if (!string.IsNullOrEmpty(queueNamePrefixAsString))
{
transport.QueueNamePrefix(queueNamePrefixAsString);
}
}
if (builder.TryGetValue("S3BucketForLargeMessages", out var bucketForLargeMessages))
{
var bucketForLargeMessagesAsString = (string)bucketForLargeMessages;
if (!string.IsNullOrEmpty(bucketForLargeMessagesAsString))
{
var keyPrefixAsString = string.Empty;
if (builder.TryGetValue("S3KeyPrefix", out var keyPrefix))
{
keyPrefixAsString = (string)keyPrefix;
}
var s3Settings = transport.S3(bucketForLargeMessagesAsString, keyPrefixAsString);
if (alwaysLoadFromEnvironmentVariable)
{
s3Settings.ClientFactory(() => new AmazonS3Client(new EnvironmentVariablesAWSCredentials()));
}
else
{
log.Info("BasicAWSCredentials have not been supplied in the connection string. Attempting to use existing environment or IAM role credentials for S3 Client.");
}
}
}
//HINT: This is needed to make Core doesn't load a connection string value from the app.config.
// This prevents SQS from throwing on startup.
var connectionString = transport.GetSettings().Get("NServiceBus.TransportConnectionString");
connectionString.GetType()
.GetField("GetValue", BindingFlags.NonPublic | BindingFlags.Instance)
?.SetValue(connectionString, (Func <string>)(() => null));
transport.Transactions(TransportTransactionMode.ReceiveOnly);
}
public void Throws_if_s3_bucket_is_specified_without_key_prefix()
{
var extensions = new TransportExtensions <SqsTransport>(new SettingsHolder());
Assert.Throws <ArgumentNullException>(() => extensions.S3("myTestBucket", string.Empty));
}