public void TestUpload()
{
const string originalFileContents = "test-data-please-ignore";
var s3Key = "/unittests/Upload-test-s3Key" + DateTime.Now.Ticks + Guid.NewGuid();
var result = TimeSpan.FromHours(1);
var mockStore = new Mock <IConfigurationStore>();
mockStore.Setup(x => x.GetValueString("AWS_TEMPORARY_BUCKET_NAME")).Returns("vss-exports-stg");
mockStore.Setup(x => x.GetValueTimeSpan("AWS_PRESIGNED_URL_EXPIRY")).Returns(result);
var transferProxy = new TransferProxy(mockStore.Object, new NullLogger <TransferProxy>(), "AWS_TEMPORARY_BUCKET_NAME");
using (var ms = new MemoryStream(Encoding.UTF8.GetBytes(originalFileContents)))
{
transferProxy.Upload(ms, s3Key);
}
var s3ResultStream = transferProxy.Download(s3Key).Result.FileStream;
var resultMemoryStream = new MemoryStream();
s3ResultStream.CopyTo(resultMemoryStream);
resultMemoryStream.Seek(0, SeekOrigin.Begin);
var text = Encoding.UTF8.GetString(resultMemoryStream.ToArray());
Assert.Equal(originalFileContents, text);
}
public T CreateExecutor <T>() where T : RequestExecutorContainer, new()
{
ConfigStore.Reset();
DataCache.Reset();
TRexTagFileProxy.Reset();
TransferProxy.Reset();
WebRequest.Reset();
return(RequestExecutorContainer.Build <T>(LoggerFactory, ConfigStore.Object, DataCache.Object, TRexTagFileProxy.Object, TransferProxyFactory.Object, WebRequest.Object));
}
public void Should_not_throw_When_reading_AWS_PRESIGNED_URL_EXPIRY(int expectedResult)
{
var result = TimeSpan.FromDays(expectedResult);
var mockStore = new Mock <IConfigurationStore>();
mockStore.Setup(x => x.GetValueString("AWS_TEMPORARY_BUCKET_NAME")).Returns("AWS_TEMPORARY_BUCKET_NAME");
mockStore.Setup(x => x.GetValueTimeSpan("AWS_PRESIGNED_URL_EXPIRY")).Returns(result);
// ReSharper disable once AssignmentIsFullyDiscarded
_ = new TransferProxy(mockStore.Object, new NullLogger <TransferProxy>(), "AWS_TEMPORARY_BUCKET_NAME");
}
public void ShouldNotUploadWhenTrexProxyReturnsInternalProcessingErrorType()
{
var e = CreateExecutor <TagFileSnsProcessExecutor>();
var theFileName = "test-filename-no-download";
var payLoad = new SnsPayload()
{
Type = SnsPayload.NotificationType,
TopicArn = "TestArn",
Message = JsonConvert.SerializeObject(new SnsTagFile()
{
Data = MockRequest.Data,
FileName = theFileName,
FileSize = MockRequest.Data.Length
})
};
var key = TagFileProcessExecutor.GetS3Key(theFileName);
var expectedS3PathOnFailure = $"{TagFileProcessExecutor.CONNECTION_ERROR_FOLDER}/{key}";
var expectedS3Path = $"{key}";
var expectedErrorCode = 3124; // Executor should forward on the error code and in this case, not archive this internal error
// Setup a failed connection
TRexTagFileProxy
.Setup(m => m.SendTagFile(It.IsAny <CompactionTagFileRequest>(),
It.IsAny <IHeaderDictionary>()))
.Returns(Task.FromResult(new ContractExecutionResult(expectedErrorCode)));
// Handle the upload
TransferProxy.Setup(m => m.Upload(It.IsAny <Stream>(), It.IsAny <string>()));
// Run the test
var result = e.ProcessAsync(payLoad).Result;
// Validate we tried to upload
TRexTagFileProxy
.Verify(m => m.SendTagFile(It.IsAny <CompactionTagFileRequest>(),
It.IsAny <IHeaderDictionary>()),
Times.Exactly(1));
// Validate that the file was not saved anywhere
TransferProxy.Verify(m => m.Upload(It.IsAny <MemoryStream>(), It.Is <string>(s => s == expectedS3Path)), Times.Never);
TransferProxy.Verify(m => m.Upload(It.IsAny <MemoryStream>(), It.Is <string>(s => s == expectedS3PathOnFailure)), Times.Never);
// Validate we got a non-zero result
result.Code.Should().Be(ContractExecutionStatesEnum.InternalProcessingError);
}
public void ShouldUploadWhenTrexProxyPasses()
{
var e = CreateExecutor <TagFileSnsProcessExecutor>();
var theFileName = "test-filename-no-download";
var payLoad = new SnsPayload()
{
Type = SnsPayload.NotificationType,
TopicArn = "TestArn",
Message = JsonConvert.SerializeObject(new SnsTagFile()
{
Data = MockRequest.Data,
FileName = theFileName,
FileSize = MockRequest.Data.Length
})
};
var key = TagFileProcessExecutor.GetS3Key(theFileName);
var expectedS3Path = $"{key}";
// Ensure the tag file will be upload and save the response
TRexTagFileProxy
.Setup(m => m.SendTagFile(It.IsAny <CompactionTagFileRequest>(),
It.IsAny <IHeaderDictionary>()))
.Callback <CompactionTagFileRequest, IHeaderDictionary>((tagFileRequest, _) => { })
.Returns(Task.FromResult(new ContractExecutionResult()));
// Handle the upload
TransferProxy.Setup(m => m.Upload(It.IsAny <Stream>(), It.IsAny <string>()));
// Run the test
var result = e.ProcessAsync(payLoad).Result;
// Validate we tried to upload
TRexTagFileProxy
.Verify(m => m.SendTagFile(It.IsAny <CompactionTagFileRequest>(),
It.IsAny <IHeaderDictionary>()),
Times.Exactly(1));
// Validate that the path was correct (we check the data separately)
TransferProxy.Verify(m => m.Upload(It.IsAny <MemoryStream>(), It.Is <string>(s => s == expectedS3Path)), Times.Once);
// Validate we got a non-zero result
result.Code.Should().Be(0);
}
public void ShouldForwardTagfile()
{
// When a SNS message comes in with data, it should be mapped to a Tag File Request and processed
var e = CreateExecutor <TagFileSnsProcessExecutor>();
CompactionTagFileRequest receivedTagFile = null;
var payLoad = new SnsPayload()
{
Type = SnsPayload.NotificationType,
TopicArn = "TestArn",
Message = JsonConvert.SerializeObject(new SnsTagFile()
{
Data = MockRequest.Data,
FileName = "test-filename-no-download",
FileSize = MockRequest.Data.Length
})
};
// Ensure the tag file will be upload and save the response
TRexTagFileProxy
.Setup(m => m.SendTagFile(It.IsAny <CompactionTagFileRequest>(),
It.IsAny <IHeaderDictionary>()))
.Callback <CompactionTagFileRequest, IHeaderDictionary>((tagFileRequest, _) => receivedTagFile = tagFileRequest)
.Returns(Task.FromResult(new ContractExecutionResult()));
// Handle the upload
TransferProxy.Setup(m => m.Upload(It.IsAny <Stream>(), It.IsAny <string>()));
var result = e.ProcessAsync(payLoad).Result;
// Validate
TRexTagFileProxy
.Verify(m => m.SendTagFile(It.IsAny <CompactionTagFileRequest>(),
It.IsAny <IHeaderDictionary>()),
Times.Once);
// Validate result and data
result.Should().NotBeNull();
result.Code.Should().Be(0);
receivedTagFile.Should().NotBeNull();
receivedTagFile.Data.Should().BeEquivalentTo(MockRequest.Data);
receivedTagFile.FileName.Should().Be("test-filename-no-download");
}
public void TestLoginNoKey()
{
// We cannot use Access keys with S3 now, so we have to use assumed roles
// This test must work without providing any credentials
const string s3BaseUrl = "https://s3.us-west-2.amazonaws.com/";
var s3Key = "Test-s3Key" + DateTime.Now.Ticks;
var mockStore = new Mock <IConfigurationStore>();
mockStore.Setup(x => x.GetValueString("AWS_TEMPORARY_BUCKET_NAME")).Returns("AWS_TEMPORARY_BUCKET_NAME");
var transferProxy = new TransferProxy(mockStore.Object, new NullLogger <TransferProxy>(), "UnitTests");
var key = transferProxy.GeneratePreSignedUrl(s3Key);
Assert.NotNull(key);
// The s3 Url will contain the key, and base url at the start.
Assert.StartsWith(s3BaseUrl + s3Key, key);
}
public void ShouldUploadWhenTrexProxyThrowsException()
{
// This simulates a situation when tRexProxy cant connect to TRex
// We want to upload the tag file to S3, but return an error to the caller
var executor = CreateExecutor <TagFileProcessExecutor>();
executor.ArchiveOnInternalError = true;
var key = TagFileProcessExecutor.GetS3Key(MockRequest.FileName);
var expectedS3Path = $"{TagFileProcessExecutor.CONNECTION_ERROR_FOLDER}/{key}";
var uploadedData = new List <byte>();
// Setup a failed connection
TRexTagFileProxy
.Setup(m => m.SendTagFile(It.IsAny <CompactionTagFileRequest>(),
It.IsAny <IHeaderDictionary>()))
.Throws <HttpRequestException>();
// Handle the upload, and save the data for validation
TransferProxy.Setup(m => m.Upload(It.IsAny <Stream>(), It.IsAny <string>()))
.Callback <Stream, string>((stream, path) => { uploadedData.AddRange(((MemoryStream)stream).ToArray()); });
// Run the test
var result = executor.ProcessAsync(MockRequest).Result;
// Validate we tried to upload
TRexTagFileProxy
.Verify(m => m.SendTagFile(It.IsAny <CompactionTagFileRequest>(),
It.IsAny <IHeaderDictionary>()),
Times.Exactly(1));
// Validate that the path was correct (we check the data separately)
TransferProxy.Verify(m => m.Upload(It.IsAny <MemoryStream>(), It.Is <string>(s => s == expectedS3Path)), Times.Once);
// Validate the data
uploadedData.Should().BeEquivalentTo(MockRequest.Data);
// Validate we got a non-zero result
result.Code.Should().NotBe(0);
}
public void ShouldUploadWhenTrexProxyFails()
{
var executor = CreateExecutor <TagFileProcessExecutor>();
executor.ArchiveOnInternalError = true;
var key = TagFileProcessExecutor.GetS3Key(MockRequest.FileName);
var expectedS3Path = $"{key}";
var uploadedData = new List <byte>();
var expectedErrorCode = 55; // Executor should forward on the error code when tRexProxy returns an error
// Setup a failed connection
TRexTagFileProxy
.Setup(m => m.SendTagFile(It.IsAny <CompactionTagFileRequest>(),
It.IsAny <IHeaderDictionary>()))
.Returns(Task.FromResult(new ContractExecutionResult(expectedErrorCode)));
// Handle the upload, and save the data for validation
TransferProxy.Setup(m => m.Upload(It.IsAny <Stream>(), It.IsAny <string>()))
.Callback <Stream, string>((stream, path) => { uploadedData.AddRange(((MemoryStream)stream).ToArray()); });
// Run the test
var result = executor.ProcessAsync(MockRequest).Result;
// Validate we tried to upload
TRexTagFileProxy
.Verify(m => m.SendTagFile(It.IsAny <CompactionTagFileRequest>(),
It.IsAny <IHeaderDictionary>()),
Times.Exactly(1));
// Validate that the path was correct (we check the data separately)
TransferProxy.Verify(m => m.Upload(It.IsAny <MemoryStream>(), It.Is <string>(s => s == expectedS3Path)), Times.Once);
// Validate the data
uploadedData.Should().BeEquivalentTo(MockRequest.Data);
// Validate we got a non-zero result
result.Code.Should().Be(expectedErrorCode);
}
public void ShouldDownloadDataWhenNeeded()
{
// When a SNS message comes in, it may contain a URL to download the file content
// This test checks that, and ensures the data is downloaded and sent as a tag file correctly
var e = CreateExecutor <TagFileSnsProcessExecutor>();
var testUrl = "http://not-a-real-host/tag";
CompactionTagFileRequest receivedTagFile = null;
var payLoad = new SnsPayload()
{
Type = SnsPayload.NotificationType,
TopicArn = "TestArn",
Message = JsonConvert.SerializeObject(new SnsTagFile()
{
DownloadUrl = testUrl,
FileName = "test-filename",
FileSize = MockRequest.Data.Length
})
};
WebRequest.Setup(m => m.ExecuteRequestAsStreamContent(It.IsAny <string>(),
It.IsAny <HttpMethod>(),
It.IsAny <IHeaderDictionary>(),
It.IsAny <Stream>(),
It.IsAny <int?>(),
It.IsAny <int>(),
It.IsAny <bool>()))
.Returns(Task.FromResult <HttpContent>(new ByteArrayContent(MockRequest.Data.ToArray())));
// Ensure the tag file will be upload and save the response
TRexTagFileProxy
.Setup(m => m.SendTagFile(It.IsAny <CompactionTagFileRequest>(),
It.IsAny <IHeaderDictionary>()))
.Callback <CompactionTagFileRequest, IHeaderDictionary>((tagFileRequest, _) => receivedTagFile = tagFileRequest)
.Returns(Task.FromResult(new ContractExecutionResult()));
// Handle the upload
TransferProxy.Setup(m => m.Upload(It.IsAny <Stream>(), It.IsAny <string>()));
var result = e.ProcessAsync(payLoad).Result;
// Validate
WebRequest.Verify(m => m.ExecuteRequestAsStreamContent(It.Is <string>(m => m == testUrl),
It.Is <HttpMethod>(m => m == HttpMethod.Get),
It.IsAny <IHeaderDictionary>(),
It.IsAny <Stream>(),
It.IsAny <int?>(),
It.IsAny <int>(),
It.IsAny <bool>()),
Times.Once);
TRexTagFileProxy
.Verify(m => m.SendTagFile(It.IsAny <CompactionTagFileRequest>(),
It.IsAny <IHeaderDictionary>()),
Times.Once);
// Validate result and data
result.Should().NotBeNull();
result.Code.Should().Be(0);
receivedTagFile.Should().NotBeNull();
receivedTagFile.Data.Should().BeEquivalentTo(MockRequest.Data);
receivedTagFile.FileName.Should().Be("test-filename");
}
