private ContractExecutionResult ProcessTagFile(CompactionTagFileRequest request, string method)
{
if (request == null)
{
_logger.LogWarning("Empty request passed");
return(ContractExecutionResult.ErrorResult("Empty Request"));
}
request.Validate();
_logger.LogInformation($"Received Tag File (via {method}) with filename: {request.FileName}. TCC Org: {request.OrgId}. Data Length: {request.Data.Length}");
using (var data = new MemoryStream(request.Data))
{
_logger.LogInformation($"Uploading Tag File {request.FileName}");
var path = GetS3Key(method, request.FileName, request.OrgId);
// S3 needs a full path including file, but TCC needs a path and filename as two separate variables
var s3FullPath = path + request.FileName;
_transferProxyFactory.NewProxy(TransferProxyType.TagFileGatewayArchive).Upload(data, s3FullPath);
_logger.LogInformation($"Successfully uploaded Tag File {request.FileName}");
}
return(new ContractExecutionResult(0));
}
protected override async Task <ContractExecutionResult> ProcessAsyncEx <T>(T item)
{
if (!(item is CompactionTagFileRequest request))
{
Logger.LogWarning($"{nameof(TagFileProcessExecutor)} Invalid Request passed in. Expected {typeof(CompactionTagFileRequest).Name} but got {(item == null ? "null" : item.GetType().Name)}");
return(ContractExecutionResult.ErrorResult("Invalid Request"));
}
request.Validate();
Logger.LogInformation($"{nameof(TagFileProcessExecutor)} Received Tag File with filename: {request.FileName}. TCC Org: {request.OrgId}. Data Length: {request.Data.Length}");
var result = ContractExecutionResult.ErrorResult("Not processed");
var internalProcessingError = false;
try
{
result = await TRexTagFileProxy.SendTagFile(request);
}
catch (Exception e)
{
Logger.LogError(e, $"{nameof(TagFileProcessExecutor)} Failed to connect to TRex. Tag file {request.FileName}");
internalProcessingError = true;
}
internalProcessingError = IsInternalError(internalProcessingError, result.Code);
// If we failed to connect to trex (or other retry-able error),
// we want to either put it separate folder or not delete from SQS que
// If the tag file was accepted, and not processed for a real reason (e.g no project found at seed position)
// then we can to archive it, as it was successfully processed with no change to the datamodel
await using (var data = new MemoryStream(request.Data))
{
Logger.LogInformation($"{nameof(TagFileProcessExecutor)} Uploading Tag File {request.FileName}");
var path = GetS3Key(request.FileName);
if (internalProcessingError)
{
path = $"{CONNECTION_ERROR_FOLDER}/{path}";
}
if (!internalProcessingError || ArchiveOnInternalError)
{
TransferProxyFactory.NewProxy(TransferProxyType.TagFileGatewayArchive).Upload(data, path);
Logger.LogInformation($"{nameof(TagFileProcessExecutor)} Successfully uploaded Tag File {request.FileName}");
}
else
{
Logger.LogInformation($"{nameof(TagFileProcessExecutor)} No S3 upload as NoArchiveOnInternalError set. Tag File {request.FileName}");
}
}
if (internalProcessingError)
{
Logger.LogError($"{nameof(TagFileProcessExecutor)} InternalProcessingError {result.Code} {request.FileName} archiveFlag: {ArchiveOnInternalError}");
return(ContractExecutionResult.ErrorResult("Failed to connect to backend"));
}
return(result);
}
protected override async Task <ContractExecutionResult> ProcessAsyncEx <T>(T item)
{
if (!(item is SnsPayload payload))
{
Logger.LogWarning($"{nameof(TagFileSnsProcessExecutor)} Invalid Request passed in. Expected {typeof(SnsPayload).Name} but got {(item == null ? "null" : item.GetType().Name)}");
return(ContractExecutionResult.ErrorResult("Invalid Request"));
}
Logger.LogInformation($"{nameof(TagFileSnsProcessExecutor)} Sns message type: {payload.Type}, topic: {payload.TopicArn}");
if (payload.Type == SnsPayload.SubscriptionType)
{
// Request for subscription
Logger.LogInformation($"{nameof(TagFileSnsProcessExecutor)} SNS SUBSCRIPTION REQUEST: {payload.Message}, Subscription URL: '{payload.SubscribeURL}'");
return(new ContractExecutionResult());
}
if (payload.IsNotification)
{
// Got a tag file
SnsTagFile tagFile;
try
{
tagFile = JsonConvert.DeserializeObject <SnsTagFile>(payload.Message);
}
catch (Exception e)
{
Logger.LogError(e, $"{nameof(TagFileSnsProcessExecutor)} Failed to deserialize SnsTagFile {payload.MessageId}");
tagFile = null;
}
if (tagFile == null)
{
// this will cause payload to be deleted from the SQS que
Logger.LogWarning($"{nameof(TagFileSnsProcessExecutor)} Could not convert to Tag File Model. JSON: {payload.Message}");
return(new ContractExecutionResult(1, "Failed to parse tag file model"));
}
byte[] data;
if (!string.IsNullOrEmpty(tagFile.DownloadUrl))
{
Logger.LogInformation($"{nameof(TagFileSnsProcessExecutor)} Tag file {tagFile.FileName} needs to be downloaded from : {tagFile.DownloadUrl}");
var downloadTagFileData = await WebRequest.ExecuteRequestAsStreamContent(tagFile.DownloadUrl, HttpMethod.Get);
await using var ms = new MemoryStream();
await downloadTagFileData.CopyToAsync(ms);
data = ms.ToArray();
if (data.Length != tagFile.FileSize)
{
Logger.LogWarning($"{nameof(TagFileSnsProcessExecutor)} Downloaded data length {data.Length} is not equal to expected length {tagFile.FileSize}");
}
Logger.LogInformation($"{nameof(TagFileSnsProcessExecutor)} Downloaded tag file {tagFile.FileName}, total bytes: {data.Length}");
}
else
{
Logger.LogInformation($"{nameof(TagFileSnsProcessExecutor)} Tag file data is included in payload for file {tagFile.FileName}");
data = tagFile.Data;
}
var request = new CompactionTagFileRequest
{
Data = data,
FileName = tagFile.FileName,
OrgId = tagFile.OrgId,
OriginSource = tagFile.originSource
};
Logger.LogInformation($"{nameof(TagFileSnsProcessExecutor)} Attempting to process sns tag file {tagFile.FileName}");
var executor = Build <TagFileProcessExecutor>();
executor.ArchiveOnInternalError = false;
var result = await executor.ProcessAsync(request);
Logger.LogInformation($"{nameof(TagFileSnsProcessExecutor)} Got result {JsonConvert.SerializeObject(result)} for Tag file: {tagFile?.FileName}");
return(result);
}
Logger.LogWarning($"{nameof(TagFileSnsProcessExecutor)} Unknown SNS Type: {payload.Type} - not sure how to process");
return(new ContractExecutionResult(99, "Unknown SNS message"));
}