protected override async Task <ContractExecutionResult> ProcessAsyncEx <T>(T item)
{
if (!(item is CompactionTagFileRequest request))
{
Logger.LogWarning($"{nameof(TagFileProcessExecutor)} Invalid Request passed in. Expected {typeof(CompactionTagFileRequest).Name} but got {(item == null ? "null" : item.GetType().Name)}");
return(ContractExecutionResult.ErrorResult("Invalid Request"));
}
request.Validate();
Logger.LogInformation($"{nameof(TagFileProcessExecutor)} Received Tag File with filename: {request.FileName}. TCC Org: {request.OrgId}. Data Length: {request.Data.Length}");
var result = ContractExecutionResult.ErrorResult("Not processed");
var internalProcessingError = false;
try
{
result = await TRexTagFileProxy.SendTagFile(request);
}
catch (Exception e)
{
Logger.LogError(e, $"{nameof(TagFileProcessExecutor)} Failed to connect to TRex. Tag file {request.FileName}");
internalProcessingError = true;
}
internalProcessingError = IsInternalError(internalProcessingError, result.Code);
// If we failed to connect to trex (or other retry-able error),
// we want to either put it separate folder or not delete from SQS que
// If the tag file was accepted, and not processed for a real reason (e.g no project found at seed position)
// then we can to archive it, as it was successfully processed with no change to the datamodel
await using (var data = new MemoryStream(request.Data))
{
Logger.LogInformation($"{nameof(TagFileProcessExecutor)} Uploading Tag File {request.FileName}");
var path = GetS3Key(request.FileName);
if (internalProcessingError)
{
path = $"{CONNECTION_ERROR_FOLDER}/{path}";
}
if (!internalProcessingError || ArchiveOnInternalError)
{
TransferProxyFactory.NewProxy(TransferProxyType.TagFileGatewayArchive).Upload(data, path);
Logger.LogInformation($"{nameof(TagFileProcessExecutor)} Successfully uploaded Tag File {request.FileName}");
}
else
{
Logger.LogInformation($"{nameof(TagFileProcessExecutor)} No S3 upload as NoArchiveOnInternalError set. Tag File {request.FileName}");
}
}
if (internalProcessingError)
{
Logger.LogError($"{nameof(TagFileProcessExecutor)} InternalProcessingError {result.Code} {request.FileName} archiveFlag: {ArchiveOnInternalError}");
return(ContractExecutionResult.ErrorResult("Failed to connect to backend"));
}
return(result);
}
public T CreateExecutor <T>() where T : RequestExecutorContainer, new()
{
ConfigStore.Reset();
DataCache.Reset();
TRexTagFileProxy.Reset();
TransferProxy.Reset();
WebRequest.Reset();
return(RequestExecutorContainer.Build <T>(LoggerFactory, ConfigStore.Object, DataCache.Object, TRexTagFileProxy.Object, TransferProxyFactory.Object, WebRequest.Object));
}
public void ShouldNotUploadWhenTrexProxyReturnsInternalProcessingErrorType()
{
var e = CreateExecutor <TagFileSnsProcessExecutor>();
var theFileName = "test-filename-no-download";
var payLoad = new SnsPayload()
{
Type = SnsPayload.NotificationType,
TopicArn = "TestArn",
Message = JsonConvert.SerializeObject(new SnsTagFile()
{
Data = MockRequest.Data,
FileName = theFileName,
FileSize = MockRequest.Data.Length
})
};
var key = TagFileProcessExecutor.GetS3Key(theFileName);
var expectedS3PathOnFailure = $"{TagFileProcessExecutor.CONNECTION_ERROR_FOLDER}/{key}";
var expectedS3Path = $"{key}";
var expectedErrorCode = 3124; // Executor should forward on the error code and in this case, not archive this internal error
// Setup a failed connection
TRexTagFileProxy
.Setup(m => m.SendTagFile(It.IsAny <CompactionTagFileRequest>(),
It.IsAny <IHeaderDictionary>()))
.Returns(Task.FromResult(new ContractExecutionResult(expectedErrorCode)));
// Handle the upload
TransferProxy.Setup(m => m.Upload(It.IsAny <Stream>(), It.IsAny <string>()));
// Run the test
var result = e.ProcessAsync(payLoad).Result;
// Validate we tried to upload
TRexTagFileProxy
.Verify(m => m.SendTagFile(It.IsAny <CompactionTagFileRequest>(),
It.IsAny <IHeaderDictionary>()),
Times.Exactly(1));
// Validate that the file was not saved anywhere
TransferProxy.Verify(m => m.Upload(It.IsAny <MemoryStream>(), It.Is <string>(s => s == expectedS3Path)), Times.Never);
TransferProxy.Verify(m => m.Upload(It.IsAny <MemoryStream>(), It.Is <string>(s => s == expectedS3PathOnFailure)), Times.Never);
// Validate we got a non-zero result
result.Code.Should().Be(ContractExecutionStatesEnum.InternalProcessingError);
}
public void ShouldUploadWhenTrexProxyPasses()
{
var e = CreateExecutor <TagFileSnsProcessExecutor>();
var theFileName = "test-filename-no-download";
var payLoad = new SnsPayload()
{
Type = SnsPayload.NotificationType,
TopicArn = "TestArn",
Message = JsonConvert.SerializeObject(new SnsTagFile()
{
Data = MockRequest.Data,
FileName = theFileName,
FileSize = MockRequest.Data.Length
})
};
var key = TagFileProcessExecutor.GetS3Key(theFileName);
var expectedS3Path = $"{key}";
// Ensure the tag file will be upload and save the response
TRexTagFileProxy
.Setup(m => m.SendTagFile(It.IsAny <CompactionTagFileRequest>(),
It.IsAny <IHeaderDictionary>()))
.Callback <CompactionTagFileRequest, IHeaderDictionary>((tagFileRequest, _) => { })
.Returns(Task.FromResult(new ContractExecutionResult()));
// Handle the upload
TransferProxy.Setup(m => m.Upload(It.IsAny <Stream>(), It.IsAny <string>()));
// Run the test
var result = e.ProcessAsync(payLoad).Result;
// Validate we tried to upload
TRexTagFileProxy
.Verify(m => m.SendTagFile(It.IsAny <CompactionTagFileRequest>(),
It.IsAny <IHeaderDictionary>()),
Times.Exactly(1));
// Validate that the path was correct (we check the data separately)
TransferProxy.Verify(m => m.Upload(It.IsAny <MemoryStream>(), It.Is <string>(s => s == expectedS3Path)), Times.Once);
// Validate we got a non-zero result
result.Code.Should().Be(0);
}
public void ShouldForwardTagfile()
{
// When a SNS message comes in with data, it should be mapped to a Tag File Request and processed
var e = CreateExecutor <TagFileSnsProcessExecutor>();
CompactionTagFileRequest receivedTagFile = null;
var payLoad = new SnsPayload()
{
Type = SnsPayload.NotificationType,
TopicArn = "TestArn",
Message = JsonConvert.SerializeObject(new SnsTagFile()
{
Data = MockRequest.Data,
FileName = "test-filename-no-download",
FileSize = MockRequest.Data.Length
})
};
// Ensure the tag file will be upload and save the response
TRexTagFileProxy
.Setup(m => m.SendTagFile(It.IsAny <CompactionTagFileRequest>(),
It.IsAny <IHeaderDictionary>()))
.Callback <CompactionTagFileRequest, IHeaderDictionary>((tagFileRequest, _) => receivedTagFile = tagFileRequest)
.Returns(Task.FromResult(new ContractExecutionResult()));
// Handle the upload
TransferProxy.Setup(m => m.Upload(It.IsAny <Stream>(), It.IsAny <string>()));
var result = e.ProcessAsync(payLoad).Result;
// Validate
TRexTagFileProxy
.Verify(m => m.SendTagFile(It.IsAny <CompactionTagFileRequest>(),
It.IsAny <IHeaderDictionary>()),
Times.Once);
// Validate result and data
result.Should().NotBeNull();
result.Code.Should().Be(0);
receivedTagFile.Should().NotBeNull();
receivedTagFile.Data.Should().BeEquivalentTo(MockRequest.Data);
receivedTagFile.FileName.Should().Be("test-filename-no-download");
}
public void ShouldUploadWhenTrexProxyThrowsException()
{
// This simulates a situation when tRexProxy cant connect to TRex
// We want to upload the tag file to S3, but return an error to the caller
var executor = CreateExecutor <TagFileProcessExecutor>();
executor.ArchiveOnInternalError = true;
var key = TagFileProcessExecutor.GetS3Key(MockRequest.FileName);
var expectedS3Path = $"{TagFileProcessExecutor.CONNECTION_ERROR_FOLDER}/{key}";
var uploadedData = new List <byte>();
// Setup a failed connection
TRexTagFileProxy
.Setup(m => m.SendTagFile(It.IsAny <CompactionTagFileRequest>(),
It.IsAny <IHeaderDictionary>()))
.Throws <HttpRequestException>();
// Handle the upload, and save the data for validation
TransferProxy.Setup(m => m.Upload(It.IsAny <Stream>(), It.IsAny <string>()))
.Callback <Stream, string>((stream, path) => { uploadedData.AddRange(((MemoryStream)stream).ToArray()); });
// Run the test
var result = executor.ProcessAsync(MockRequest).Result;
// Validate we tried to upload
TRexTagFileProxy
.Verify(m => m.SendTagFile(It.IsAny <CompactionTagFileRequest>(),
It.IsAny <IHeaderDictionary>()),
Times.Exactly(1));
// Validate that the path was correct (we check the data separately)
TransferProxy.Verify(m => m.Upload(It.IsAny <MemoryStream>(), It.Is <string>(s => s == expectedS3Path)), Times.Once);
// Validate the data
uploadedData.Should().BeEquivalentTo(MockRequest.Data);
// Validate we got a non-zero result
result.Code.Should().NotBe(0);
}
public void ShouldUploadWhenTrexProxyFails()
{
var executor = CreateExecutor <TagFileProcessExecutor>();
executor.ArchiveOnInternalError = true;
var key = TagFileProcessExecutor.GetS3Key(MockRequest.FileName);
var expectedS3Path = $"{key}";
var uploadedData = new List <byte>();
var expectedErrorCode = 55; // Executor should forward on the error code when tRexProxy returns an error
// Setup a failed connection
TRexTagFileProxy
.Setup(m => m.SendTagFile(It.IsAny <CompactionTagFileRequest>(),
It.IsAny <IHeaderDictionary>()))
.Returns(Task.FromResult(new ContractExecutionResult(expectedErrorCode)));
// Handle the upload, and save the data for validation
TransferProxy.Setup(m => m.Upload(It.IsAny <Stream>(), It.IsAny <string>()))
.Callback <Stream, string>((stream, path) => { uploadedData.AddRange(((MemoryStream)stream).ToArray()); });
// Run the test
var result = executor.ProcessAsync(MockRequest).Result;
// Validate we tried to upload
TRexTagFileProxy
.Verify(m => m.SendTagFile(It.IsAny <CompactionTagFileRequest>(),
It.IsAny <IHeaderDictionary>()),
Times.Exactly(1));
// Validate that the path was correct (we check the data separately)
TransferProxy.Verify(m => m.Upload(It.IsAny <MemoryStream>(), It.Is <string>(s => s == expectedS3Path)), Times.Once);
// Validate the data
uploadedData.Should().BeEquivalentTo(MockRequest.Data);
// Validate we got a non-zero result
result.Code.Should().Be(expectedErrorCode);
}
public void ShouldDownloadDataWhenNeeded()
{
// When a SNS message comes in, it may contain a URL to download the file content
// This test checks that, and ensures the data is downloaded and sent as a tag file correctly
var e = CreateExecutor <TagFileSnsProcessExecutor>();
var testUrl = "http://not-a-real-host/tag";
CompactionTagFileRequest receivedTagFile = null;
var payLoad = new SnsPayload()
{
Type = SnsPayload.NotificationType,
TopicArn = "TestArn",
Message = JsonConvert.SerializeObject(new SnsTagFile()
{
DownloadUrl = testUrl,
FileName = "test-filename",
FileSize = MockRequest.Data.Length
})
};
WebRequest.Setup(m => m.ExecuteRequestAsStreamContent(It.IsAny <string>(),
It.IsAny <HttpMethod>(),
It.IsAny <IHeaderDictionary>(),
It.IsAny <Stream>(),
It.IsAny <int?>(),
It.IsAny <int>(),
It.IsAny <bool>()))
.Returns(Task.FromResult <HttpContent>(new ByteArrayContent(MockRequest.Data.ToArray())));
// Ensure the tag file will be upload and save the response
TRexTagFileProxy
.Setup(m => m.SendTagFile(It.IsAny <CompactionTagFileRequest>(),
It.IsAny <IHeaderDictionary>()))
.Callback <CompactionTagFileRequest, IHeaderDictionary>((tagFileRequest, _) => receivedTagFile = tagFileRequest)
.Returns(Task.FromResult(new ContractExecutionResult()));
// Handle the upload
TransferProxy.Setup(m => m.Upload(It.IsAny <Stream>(), It.IsAny <string>()));
var result = e.ProcessAsync(payLoad).Result;
// Validate
WebRequest.Verify(m => m.ExecuteRequestAsStreamContent(It.Is <string>(m => m == testUrl),
It.Is <HttpMethod>(m => m == HttpMethod.Get),
It.IsAny <IHeaderDictionary>(),
It.IsAny <Stream>(),
It.IsAny <int?>(),
It.IsAny <int>(),
It.IsAny <bool>()),
Times.Once);
TRexTagFileProxy
.Verify(m => m.SendTagFile(It.IsAny <CompactionTagFileRequest>(),
It.IsAny <IHeaderDictionary>()),
Times.Once);
// Validate result and data
result.Should().NotBeNull();
result.Code.Should().Be(0);
receivedTagFile.Should().NotBeNull();
receivedTagFile.Data.Should().BeEquivalentTo(MockRequest.Data);
receivedTagFile.FileName.Should().Be("test-filename");
}
