public async Task <ContractExecutionResult> SendTagFile(CompactionTagFileRequest compactionTagFileRequest, string prodHost, string prodBase)
{
var route = prodBase + TAGFILE_ROUTE;
var client = GetClient(prodHost);
if (client == null)
{
return(new ContractExecutionResult(-1, "Failed to get HttpClient"));
}
string jsonTxt = JsonConvert.SerializeObject(compactionTagFileRequest);
HttpContent requestContent = new StringContent(jsonTxt, Encoding.UTF8, "application/json");
try
{
HttpResponseMessage responseMessage = await client.PostAsync(route, requestContent);
var result = await responseMessage.Content.ReadAsStringAsync();
if (responseMessage.StatusCode == HttpStatusCode.OK)
{
return(new ContractExecutionResult());
}
else
{
return(new ContractExecutionResult(ContractExecutionStatesEnum.InternalProcessingError, result));
}
}
catch (Exception e)
{
return(new ContractExecutionResult(ContractExecutionStatesEnum.InternalProcessingError, e.Message));
}
}
public override string CacheLifeKey => "TREX_TAGFILE_CACHE_LIFE"; // not used
public async Task <ContractExecutionResult> SendTagFile(CompactionTagFileRequest compactionTagFileRequest,
IHeaderDictionary customHeaders = null)
{
log.LogDebug($"{nameof(SendTagFile)}: Filename: {compactionTagFileRequest.FileName}");
Gateway = GatewayType.Mutable;
return(await SendTagFileRequest(compactionTagFileRequest, customHeaders, "/tagfiles"));
}
private async Task <ContractExecutionResult> UploadFile(string filename)
{
var prodHost = _config.GetValueString("production-host") ?? TagConstants.DEFAULT_HOST;
var prodBase = _config.GetValueString("production-base") ?? TagConstants.DEFAULT_BASE;
_log.LogInformation($"Uploading Tagfile {filename}. Host:{prodHost}, Base:{prodBase}");
var fileData = File.ReadAllBytes(filename);
var compactionTagFileRequest = new CompactionTagFileRequest
{
FileName = Path.GetFileName(filename),
Data = fileData
};
try
{
var res = await tagFileTransfer.SendTagFile(compactionTagFileRequest, prodHost, prodBase);
_log.LogInformation($"Tagfile upload result: {res.Message}");
return(res);
}
catch (Exception e)
{
_log.LogError(e, $"Can not submit file {filename}");
return(new ContractExecutionResult(ContractExecutionStatesEnum.InternalProcessingError, e.Message));
}
}
private ContractExecutionResult ProcessTagFile(CompactionTagFileRequest request, string method)
{
if (request == null)
{
_logger.LogWarning("Empty request passed");
return(ContractExecutionResult.ErrorResult("Empty Request"));
}
request.Validate();
_logger.LogInformation($"Received Tag File (via {method}) with filename: {request.FileName}. TCC Org: {request.OrgId}. Data Length: {request.Data.Length}");
using (var data = new MemoryStream(request.Data))
{
_logger.LogInformation($"Uploading Tag File {request.FileName}");
var path = GetS3Key(method, request.FileName, request.OrgId);
// S3 needs a full path including file, but TCC needs a path and filename as two separate variables
var s3FullPath = path + request.FileName;
_transferProxyFactory.NewProxy(TransferProxyType.TagFileGatewayArchive).Upload(data, s3FullPath);
_logger.LogInformation($"Successfully uploaded Tag File {request.FileName}");
}
return(new ContractExecutionResult(0));
}
public async Task <ContractExecutionResult> PostTagFile([FromBody] CompactionTagFileRequest request,
[FromServices] ILoggerFactory loggerFactory,
[FromServices] IConfigurationStore configStore,
[FromServices] IDataCache dataCache,
[FromServices] ITRexTagFileProxy trexTagFileProxy,
[FromServices] ITransferProxyFactory transferProxyFactory,
[FromServices] IWebRequest webRequest)
{
var isDirect = Request.Path.Value.Contains("/direct");
_logger.LogInformation($"{nameof(PostTagFile)} Attempting to process {(isDirect ? "Direct" : "Non-Direct")} tag file {request?.FileName}");
var executor = RequestExecutorContainer
.Build <TagFileProcessExecutor>(loggerFactory, configStore, dataCache, trexTagFileProxy, transferProxyFactory, webRequest);
executor.ArchiveOnInternalError = true;
var result = await executor.ProcessAsync(request);
_logger.LogInformation($"{nameof(PostTagFile)} Got result {JsonConvert.SerializeObject(result)} for Tag file: {request?.FileName}");
// If we uploaded, return a successful result
// (as the tag file may not have been processed for legitimate reasons)
// We don't want the machine sending tag files over and over again in this instance
return(new ContractExecutionResult());
}
public virtual async Task <ContractExecutionResult> SendTagFileRequest(CompactionTagFileRequest compactionTagFileRequest,
IHeaderDictionary customHeaders, string route)
{
var jsonData = JsonConvert.SerializeObject(compactionTagFileRequest);
using (var payload = new MemoryStream(Encoding.UTF8.GetBytes(jsonData)))
return(await MasterDataItemServiceDiscoveryNoCache <ContractExecutionResult>(route, customHeaders, HttpMethod.Post, payload : payload));
}
public Task <ContractExecutionResult> PostTagFile([FromBody] CompactionTagFileRequest request)
{
var serializedRequest = ConvertObjectForLogging.SerializeObjectIgnoringProperties(request, "Data");
Log.LogInformation("PostTagFile: " + serializedRequest);
return(WithServiceExceptionTryExecuteAsync(() => RequestExecutorContainer
.Build <TagFileExecutor>(ConfigStore, LoggerFactory, ServiceExceptionHandler)
.ProcessAsync(request)));
}
/// <summary>
/// Sends tag file to TRex endpoint, retrieving result
/// </summary>
public static async Task <ContractExecutionResult> SendTagFileToTRex(CompactionTagFileRequest compactionTagFileRequest,
ITRexTagFileProxy tagFileProxy,
ILogger log, IHeaderDictionary customHeaders)
{
try
{
return(await tagFileProxy.SendTagFile(compactionTagFileRequest, customHeaders));
}
catch (Exception e)
{
log.LogError(e, $"{nameof(SendTagFileToTRex)}: returned exception");
}
return(new ContractExecutionResult((int)TRexTagFileResultCode.TRexUnknownException));
}
/// <summary>
/// Static constructor.
/// </summary>
public static CompactionTagFileRequestExtended CreateCompactionTagFileRequestExtended(
CompactionTagFileRequest compactionTagFileRequest,
WGS84Fence boundary,
bool includeTrexIfConfigured = true)
{
return(new CompactionTagFileRequestExtended
{
FileName = compactionTagFileRequest.FileName,
Data = compactionTagFileRequest.Data,
OrgId = compactionTagFileRequest.OrgId,
ProjectId = compactionTagFileRequest.ProjectId,
ProjectUid = compactionTagFileRequest.ProjectUid,
Boundary = boundary,
includeTrexIfConfigured = includeTrexIfConfigured
});
}
public void ShouldForwardTagfile()
{
// When a SNS message comes in with data, it should be mapped to a Tag File Request and processed
var e = CreateExecutor <TagFileSnsProcessExecutor>();
CompactionTagFileRequest receivedTagFile = null;
var payLoad = new SnsPayload()
{
Type = SnsPayload.NotificationType,
TopicArn = "TestArn",
Message = JsonConvert.SerializeObject(new SnsTagFile()
{
Data = MockRequest.Data,
FileName = "test-filename-no-download",
FileSize = MockRequest.Data.Length
})
};
// Ensure the tag file will be upload and save the response
TRexTagFileProxy
.Setup(m => m.SendTagFile(It.IsAny <CompactionTagFileRequest>(),
It.IsAny <IHeaderDictionary>()))
.Callback <CompactionTagFileRequest, IHeaderDictionary>((tagFileRequest, _) => receivedTagFile = tagFileRequest)
.Returns(Task.FromResult(new ContractExecutionResult()));
// Handle the upload
TransferProxy.Setup(m => m.Upload(It.IsAny <Stream>(), It.IsAny <string>()));
var result = e.ProcessAsync(payLoad).Result;
// Validate
TRexTagFileProxy
.Verify(m => m.SendTagFile(It.IsAny <CompactionTagFileRequest>(),
It.IsAny <IHeaderDictionary>()),
Times.Once);
// Validate result and data
result.Should().NotBeNull();
result.Code.Should().Be(0);
receivedTagFile.Should().NotBeNull();
receivedTagFile.Data.Should().BeEquivalentTo(MockRequest.Data);
receivedTagFile.FileName.Should().Be("test-filename-no-download");
}
public async Task <IActionResult> PostTagFileManualImport([FromBody] CompactionTagFileRequest request)
{
var serializedRequest = JsonUtilities.SerializeObjectIgnoringProperties(request, "Data");
_log.LogDebug($"{nameof(PostTagFileManualImport)}: request {serializedRequest}");
await ValidateRequest(request.ProjectUid);
//Now submit tag file to TRex
var requestExt = CompactionTagFileRequestExtended.CreateCompactionTagFileRequestExtended(request, null);
var responseObj = await RequestExecutorContainerFactory
.Build <TagFileSubmissionExecutor>(_logger,
_configStore, tRexTagFileProxy : _tRexTagFileProxy, customHeaders : CustomHeaders).ProcessAsync(requestExt);
return(responseObj.Code == 0
? (IActionResult)Ok(responseObj)
: BadRequest(responseObj));
}
public void ShouldDownloadDataWhenNeeded()
{
// When a SNS message comes in, it may contain a URL to download the file content
// This test checks that, and ensures the data is downloaded and sent as a tag file correctly
var e = CreateExecutor <TagFileSnsProcessExecutor>();
var testUrl = "http://not-a-real-host/tag";
CompactionTagFileRequest receivedTagFile = null;
var payLoad = new SnsPayload()
{
Type = SnsPayload.NotificationType,
TopicArn = "TestArn",
Message = JsonConvert.SerializeObject(new SnsTagFile()
{
DownloadUrl = testUrl,
FileName = "test-filename",
FileSize = MockRequest.Data.Length
})
};
WebRequest.Setup(m => m.ExecuteRequestAsStreamContent(It.IsAny <string>(),
It.IsAny <HttpMethod>(),
It.IsAny <IHeaderDictionary>(),
It.IsAny <Stream>(),
It.IsAny <int?>(),
It.IsAny <int>(),
It.IsAny <bool>()))
.Returns(Task.FromResult <HttpContent>(new ByteArrayContent(MockRequest.Data.ToArray())));
// Ensure the tag file will be upload and save the response
TRexTagFileProxy
.Setup(m => m.SendTagFile(It.IsAny <CompactionTagFileRequest>(),
It.IsAny <IHeaderDictionary>()))
.Callback <CompactionTagFileRequest, IHeaderDictionary>((tagFileRequest, _) => receivedTagFile = tagFileRequest)
.Returns(Task.FromResult(new ContractExecutionResult()));
// Handle the upload
TransferProxy.Setup(m => m.Upload(It.IsAny <Stream>(), It.IsAny <string>()));
var result = e.ProcessAsync(payLoad).Result;
// Validate
WebRequest.Verify(m => m.ExecuteRequestAsStreamContent(It.Is <string>(m => m == testUrl),
It.Is <HttpMethod>(m => m == HttpMethod.Get),
It.IsAny <IHeaderDictionary>(),
It.IsAny <Stream>(),
It.IsAny <int?>(),
It.IsAny <int>(),
It.IsAny <bool>()),
Times.Once);
TRexTagFileProxy
.Verify(m => m.SendTagFile(It.IsAny <CompactionTagFileRequest>(),
It.IsAny <IHeaderDictionary>()),
Times.Once);
// Validate result and data
result.Should().NotBeNull();
result.Code.Should().Be(0);
receivedTagFile.Should().NotBeNull();
receivedTagFile.Data.Should().BeEquivalentTo(MockRequest.Data);
receivedTagFile.FileName.Should().Be("test-filename");
}
public IActionResult PostTagFileDirectSubmission([FromBody] CompactionTagFileRequest request)
{
return(Json(ProcessTagFile(request, "direct")));
}
protected override async Task <ContractExecutionResult> ProcessAsyncEx <T>(T item)
{
if (!(item is SnsPayload payload))
{
Logger.LogWarning($"{nameof(TagFileSnsProcessExecutor)} Invalid Request passed in. Expected {typeof(SnsPayload).Name} but got {(item == null ? "null" : item.GetType().Name)}");
return(ContractExecutionResult.ErrorResult("Invalid Request"));
}
Logger.LogInformation($"{nameof(TagFileSnsProcessExecutor)} Sns message type: {payload.Type}, topic: {payload.TopicArn}");
if (payload.Type == SnsPayload.SubscriptionType)
{
// Request for subscription
Logger.LogInformation($"{nameof(TagFileSnsProcessExecutor)} SNS SUBSCRIPTION REQUEST: {payload.Message}, Subscription URL: '{payload.SubscribeURL}'");
return(new ContractExecutionResult());
}
if (payload.IsNotification)
{
// Got a tag file
SnsTagFile tagFile;
try
{
tagFile = JsonConvert.DeserializeObject <SnsTagFile>(payload.Message);
}
catch (Exception e)
{
Logger.LogError(e, $"{nameof(TagFileSnsProcessExecutor)} Failed to deserialize SnsTagFile {payload.MessageId}");
tagFile = null;
}
if (tagFile == null)
{
// this will cause payload to be deleted from the SQS que
Logger.LogWarning($"{nameof(TagFileSnsProcessExecutor)} Could not convert to Tag File Model. JSON: {payload.Message}");
return(new ContractExecutionResult(1, "Failed to parse tag file model"));
}
byte[] data;
if (!string.IsNullOrEmpty(tagFile.DownloadUrl))
{
Logger.LogInformation($"{nameof(TagFileSnsProcessExecutor)} Tag file {tagFile.FileName} needs to be downloaded from : {tagFile.DownloadUrl}");
var downloadTagFileData = await WebRequest.ExecuteRequestAsStreamContent(tagFile.DownloadUrl, HttpMethod.Get);
await using var ms = new MemoryStream();
await downloadTagFileData.CopyToAsync(ms);
data = ms.ToArray();
if (data.Length != tagFile.FileSize)
{
Logger.LogWarning($"{nameof(TagFileSnsProcessExecutor)} Downloaded data length {data.Length} is not equal to expected length {tagFile.FileSize}");
}
Logger.LogInformation($"{nameof(TagFileSnsProcessExecutor)} Downloaded tag file {tagFile.FileName}, total bytes: {data.Length}");
}
else
{
Logger.LogInformation($"{nameof(TagFileSnsProcessExecutor)} Tag file data is included in payload for file {tagFile.FileName}");
data = tagFile.Data;
}
var request = new CompactionTagFileRequest
{
Data = data,
FileName = tagFile.FileName,
OrgId = tagFile.OrgId,
OriginSource = tagFile.originSource
};
Logger.LogInformation($"{nameof(TagFileSnsProcessExecutor)} Attempting to process sns tag file {tagFile.FileName}");
var executor = Build <TagFileProcessExecutor>();
executor.ArchiveOnInternalError = false;
var result = await executor.ProcessAsync(request);
Logger.LogInformation($"{nameof(TagFileSnsProcessExecutor)} Got result {JsonConvert.SerializeObject(result)} for Tag file: {tagFile?.FileName}");
return(result);
}
Logger.LogWarning($"{nameof(TagFileSnsProcessExecutor)} Unknown SNS Type: {payload.Type} - not sure how to process");
return(new ContractExecutionResult(99, "Unknown SNS message"));
}
public ContractExecutionResult SendTagFile([FromBody] CompactionTagFileRequest compactionTagFileRequest)
{
Logger.LogInformation($"{nameof(SendTagFile)}: CompactionTagFileRequest {JsonConvert.SerializeObject(compactionTagFileRequest)}");
return(new ContractExecutionResult());
}
