public async Task <ContractExecutionResult> SendTagFile(CompactionTagFileRequest compactionTagFileRequest, string prodHost, string prodBase) { var route = prodBase + TAGFILE_ROUTE; var client = GetClient(prodHost); if (client == null) { return(new ContractExecutionResult(-1, "Failed to get HttpClient")); } string jsonTxt = JsonConvert.SerializeObject(compactionTagFileRequest); HttpContent requestContent = new StringContent(jsonTxt, Encoding.UTF8, "application/json"); try { HttpResponseMessage responseMessage = await client.PostAsync(route, requestContent); var result = await responseMessage.Content.ReadAsStringAsync(); if (responseMessage.StatusCode == HttpStatusCode.OK) { return(new ContractExecutionResult()); } else { return(new ContractExecutionResult(ContractExecutionStatesEnum.InternalProcessingError, result)); } } catch (Exception e) { return(new ContractExecutionResult(ContractExecutionStatesEnum.InternalProcessingError, e.Message)); } }
public override string CacheLifeKey => "TREX_TAGFILE_CACHE_LIFE"; // not used public async Task <ContractExecutionResult> SendTagFile(CompactionTagFileRequest compactionTagFileRequest, IHeaderDictionary customHeaders = null) { log.LogDebug($"{nameof(SendTagFile)}: Filename: {compactionTagFileRequest.FileName}"); Gateway = GatewayType.Mutable; return(await SendTagFileRequest(compactionTagFileRequest, customHeaders, "/tagfiles")); }
private async Task <ContractExecutionResult> UploadFile(string filename) { var prodHost = _config.GetValueString("production-host") ?? TagConstants.DEFAULT_HOST; var prodBase = _config.GetValueString("production-base") ?? TagConstants.DEFAULT_BASE; _log.LogInformation($"Uploading Tagfile {filename}. Host:{prodHost}, Base:{prodBase}"); var fileData = File.ReadAllBytes(filename); var compactionTagFileRequest = new CompactionTagFileRequest { FileName = Path.GetFileName(filename), Data = fileData }; try { var res = await tagFileTransfer.SendTagFile(compactionTagFileRequest, prodHost, prodBase); _log.LogInformation($"Tagfile upload result: {res.Message}"); return(res); } catch (Exception e) { _log.LogError(e, $"Can not submit file {filename}"); return(new ContractExecutionResult(ContractExecutionStatesEnum.InternalProcessingError, e.Message)); } }
private ContractExecutionResult ProcessTagFile(CompactionTagFileRequest request, string method) { if (request == null) { _logger.LogWarning("Empty request passed"); return(ContractExecutionResult.ErrorResult("Empty Request")); } request.Validate(); _logger.LogInformation($"Received Tag File (via {method}) with filename: {request.FileName}. TCC Org: {request.OrgId}. Data Length: {request.Data.Length}"); using (var data = new MemoryStream(request.Data)) { _logger.LogInformation($"Uploading Tag File {request.FileName}"); var path = GetS3Key(method, request.FileName, request.OrgId); // S3 needs a full path including file, but TCC needs a path and filename as two separate variables var s3FullPath = path + request.FileName; _transferProxyFactory.NewProxy(TransferProxyType.TagFileGatewayArchive).Upload(data, s3FullPath); _logger.LogInformation($"Successfully uploaded Tag File {request.FileName}"); } return(new ContractExecutionResult(0)); }
public async Task <ContractExecutionResult> PostTagFile([FromBody] CompactionTagFileRequest request, [FromServices] ILoggerFactory loggerFactory, [FromServices] IConfigurationStore configStore, [FromServices] IDataCache dataCache, [FromServices] ITRexTagFileProxy trexTagFileProxy, [FromServices] ITransferProxyFactory transferProxyFactory, [FromServices] IWebRequest webRequest) { var isDirect = Request.Path.Value.Contains("/direct"); _logger.LogInformation($"{nameof(PostTagFile)} Attempting to process {(isDirect ? "Direct" : "Non-Direct")} tag file {request?.FileName}"); var executor = RequestExecutorContainer .Build <TagFileProcessExecutor>(loggerFactory, configStore, dataCache, trexTagFileProxy, transferProxyFactory, webRequest); executor.ArchiveOnInternalError = true; var result = await executor.ProcessAsync(request); _logger.LogInformation($"{nameof(PostTagFile)} Got result {JsonConvert.SerializeObject(result)} for Tag file: {request?.FileName}"); // If we uploaded, return a successful result // (as the tag file may not have been processed for legitimate reasons) // We don't want the machine sending tag files over and over again in this instance return(new ContractExecutionResult()); }
public virtual async Task <ContractExecutionResult> SendTagFileRequest(CompactionTagFileRequest compactionTagFileRequest, IHeaderDictionary customHeaders, string route) { var jsonData = JsonConvert.SerializeObject(compactionTagFileRequest); using (var payload = new MemoryStream(Encoding.UTF8.GetBytes(jsonData))) return(await MasterDataItemServiceDiscoveryNoCache <ContractExecutionResult>(route, customHeaders, HttpMethod.Post, payload : payload)); }
public Task <ContractExecutionResult> PostTagFile([FromBody] CompactionTagFileRequest request) { var serializedRequest = ConvertObjectForLogging.SerializeObjectIgnoringProperties(request, "Data"); Log.LogInformation("PostTagFile: " + serializedRequest); return(WithServiceExceptionTryExecuteAsync(() => RequestExecutorContainer .Build <TagFileExecutor>(ConfigStore, LoggerFactory, ServiceExceptionHandler) .ProcessAsync(request))); }
/// <summary> /// Sends tag file to TRex endpoint, retrieving result /// </summary> public static async Task <ContractExecutionResult> SendTagFileToTRex(CompactionTagFileRequest compactionTagFileRequest, ITRexTagFileProxy tagFileProxy, ILogger log, IHeaderDictionary customHeaders) { try { return(await tagFileProxy.SendTagFile(compactionTagFileRequest, customHeaders)); } catch (Exception e) { log.LogError(e, $"{nameof(SendTagFileToTRex)}: returned exception"); } return(new ContractExecutionResult((int)TRexTagFileResultCode.TRexUnknownException)); }
/// <summary> /// Static constructor. /// </summary> public static CompactionTagFileRequestExtended CreateCompactionTagFileRequestExtended( CompactionTagFileRequest compactionTagFileRequest, WGS84Fence boundary, bool includeTrexIfConfigured = true) { return(new CompactionTagFileRequestExtended { FileName = compactionTagFileRequest.FileName, Data = compactionTagFileRequest.Data, OrgId = compactionTagFileRequest.OrgId, ProjectId = compactionTagFileRequest.ProjectId, ProjectUid = compactionTagFileRequest.ProjectUid, Boundary = boundary, includeTrexIfConfigured = includeTrexIfConfigured }); }
public void ShouldForwardTagfile() { // When a SNS message comes in with data, it should be mapped to a Tag File Request and processed var e = CreateExecutor <TagFileSnsProcessExecutor>(); CompactionTagFileRequest receivedTagFile = null; var payLoad = new SnsPayload() { Type = SnsPayload.NotificationType, TopicArn = "TestArn", Message = JsonConvert.SerializeObject(new SnsTagFile() { Data = MockRequest.Data, FileName = "test-filename-no-download", FileSize = MockRequest.Data.Length }) }; // Ensure the tag file will be upload and save the response TRexTagFileProxy .Setup(m => m.SendTagFile(It.IsAny <CompactionTagFileRequest>(), It.IsAny <IHeaderDictionary>())) .Callback <CompactionTagFileRequest, IHeaderDictionary>((tagFileRequest, _) => receivedTagFile = tagFileRequest) .Returns(Task.FromResult(new ContractExecutionResult())); // Handle the upload TransferProxy.Setup(m => m.Upload(It.IsAny <Stream>(), It.IsAny <string>())); var result = e.ProcessAsync(payLoad).Result; // Validate TRexTagFileProxy .Verify(m => m.SendTagFile(It.IsAny <CompactionTagFileRequest>(), It.IsAny <IHeaderDictionary>()), Times.Once); // Validate result and data result.Should().NotBeNull(); result.Code.Should().Be(0); receivedTagFile.Should().NotBeNull(); receivedTagFile.Data.Should().BeEquivalentTo(MockRequest.Data); receivedTagFile.FileName.Should().Be("test-filename-no-download"); }
public async Task <IActionResult> PostTagFileManualImport([FromBody] CompactionTagFileRequest request) { var serializedRequest = JsonUtilities.SerializeObjectIgnoringProperties(request, "Data"); _log.LogDebug($"{nameof(PostTagFileManualImport)}: request {serializedRequest}"); await ValidateRequest(request.ProjectUid); //Now submit tag file to TRex var requestExt = CompactionTagFileRequestExtended.CreateCompactionTagFileRequestExtended(request, null); var responseObj = await RequestExecutorContainerFactory .Build <TagFileSubmissionExecutor>(_logger, _configStore, tRexTagFileProxy : _tRexTagFileProxy, customHeaders : CustomHeaders).ProcessAsync(requestExt); return(responseObj.Code == 0 ? (IActionResult)Ok(responseObj) : BadRequest(responseObj)); }
public void ShouldDownloadDataWhenNeeded() { // When a SNS message comes in, it may contain a URL to download the file content // This test checks that, and ensures the data is downloaded and sent as a tag file correctly var e = CreateExecutor <TagFileSnsProcessExecutor>(); var testUrl = "http://not-a-real-host/tag"; CompactionTagFileRequest receivedTagFile = null; var payLoad = new SnsPayload() { Type = SnsPayload.NotificationType, TopicArn = "TestArn", Message = JsonConvert.SerializeObject(new SnsTagFile() { DownloadUrl = testUrl, FileName = "test-filename", FileSize = MockRequest.Data.Length }) }; WebRequest.Setup(m => m.ExecuteRequestAsStreamContent(It.IsAny <string>(), It.IsAny <HttpMethod>(), It.IsAny <IHeaderDictionary>(), It.IsAny <Stream>(), It.IsAny <int?>(), It.IsAny <int>(), It.IsAny <bool>())) .Returns(Task.FromResult <HttpContent>(new ByteArrayContent(MockRequest.Data.ToArray()))); // Ensure the tag file will be upload and save the response TRexTagFileProxy .Setup(m => m.SendTagFile(It.IsAny <CompactionTagFileRequest>(), It.IsAny <IHeaderDictionary>())) .Callback <CompactionTagFileRequest, IHeaderDictionary>((tagFileRequest, _) => receivedTagFile = tagFileRequest) .Returns(Task.FromResult(new ContractExecutionResult())); // Handle the upload TransferProxy.Setup(m => m.Upload(It.IsAny <Stream>(), It.IsAny <string>())); var result = e.ProcessAsync(payLoad).Result; // Validate WebRequest.Verify(m => m.ExecuteRequestAsStreamContent(It.Is <string>(m => m == testUrl), It.Is <HttpMethod>(m => m == HttpMethod.Get), It.IsAny <IHeaderDictionary>(), It.IsAny <Stream>(), It.IsAny <int?>(), It.IsAny <int>(), It.IsAny <bool>()), Times.Once); TRexTagFileProxy .Verify(m => m.SendTagFile(It.IsAny <CompactionTagFileRequest>(), It.IsAny <IHeaderDictionary>()), Times.Once); // Validate result and data result.Should().NotBeNull(); result.Code.Should().Be(0); receivedTagFile.Should().NotBeNull(); receivedTagFile.Data.Should().BeEquivalentTo(MockRequest.Data); receivedTagFile.FileName.Should().Be("test-filename"); }
public IActionResult PostTagFileDirectSubmission([FromBody] CompactionTagFileRequest request) { return(Json(ProcessTagFile(request, "direct"))); }
protected override async Task <ContractExecutionResult> ProcessAsyncEx <T>(T item) { if (!(item is SnsPayload payload)) { Logger.LogWarning($"{nameof(TagFileSnsProcessExecutor)} Invalid Request passed in. Expected {typeof(SnsPayload).Name} but got {(item == null ? "null" : item.GetType().Name)}"); return(ContractExecutionResult.ErrorResult("Invalid Request")); } Logger.LogInformation($"{nameof(TagFileSnsProcessExecutor)} Sns message type: {payload.Type}, topic: {payload.TopicArn}"); if (payload.Type == SnsPayload.SubscriptionType) { // Request for subscription Logger.LogInformation($"{nameof(TagFileSnsProcessExecutor)} SNS SUBSCRIPTION REQUEST: {payload.Message}, Subscription URL: '{payload.SubscribeURL}'"); return(new ContractExecutionResult()); } if (payload.IsNotification) { // Got a tag file SnsTagFile tagFile; try { tagFile = JsonConvert.DeserializeObject <SnsTagFile>(payload.Message); } catch (Exception e) { Logger.LogError(e, $"{nameof(TagFileSnsProcessExecutor)} Failed to deserialize SnsTagFile {payload.MessageId}"); tagFile = null; } if (tagFile == null) { // this will cause payload to be deleted from the SQS que Logger.LogWarning($"{nameof(TagFileSnsProcessExecutor)} Could not convert to Tag File Model. JSON: {payload.Message}"); return(new ContractExecutionResult(1, "Failed to parse tag file model")); } byte[] data; if (!string.IsNullOrEmpty(tagFile.DownloadUrl)) { Logger.LogInformation($"{nameof(TagFileSnsProcessExecutor)} Tag file {tagFile.FileName} needs to be downloaded from : {tagFile.DownloadUrl}"); var downloadTagFileData = await WebRequest.ExecuteRequestAsStreamContent(tagFile.DownloadUrl, HttpMethod.Get); await using var ms = new MemoryStream(); await downloadTagFileData.CopyToAsync(ms); data = ms.ToArray(); if (data.Length != tagFile.FileSize) { Logger.LogWarning($"{nameof(TagFileSnsProcessExecutor)} Downloaded data length {data.Length} is not equal to expected length {tagFile.FileSize}"); } Logger.LogInformation($"{nameof(TagFileSnsProcessExecutor)} Downloaded tag file {tagFile.FileName}, total bytes: {data.Length}"); } else { Logger.LogInformation($"{nameof(TagFileSnsProcessExecutor)} Tag file data is included in payload for file {tagFile.FileName}"); data = tagFile.Data; } var request = new CompactionTagFileRequest { Data = data, FileName = tagFile.FileName, OrgId = tagFile.OrgId, OriginSource = tagFile.originSource }; Logger.LogInformation($"{nameof(TagFileSnsProcessExecutor)} Attempting to process sns tag file {tagFile.FileName}"); var executor = Build <TagFileProcessExecutor>(); executor.ArchiveOnInternalError = false; var result = await executor.ProcessAsync(request); Logger.LogInformation($"{nameof(TagFileSnsProcessExecutor)} Got result {JsonConvert.SerializeObject(result)} for Tag file: {tagFile?.FileName}"); return(result); } Logger.LogWarning($"{nameof(TagFileSnsProcessExecutor)} Unknown SNS Type: {payload.Type} - not sure how to process"); return(new ContractExecutionResult(99, "Unknown SNS message")); }
public ContractExecutionResult SendTagFile([FromBody] CompactionTagFileRequest compactionTagFileRequest) { Logger.LogInformation($"{nameof(SendTagFile)}: CompactionTagFileRequest {JsonConvert.SerializeObject(compactionTagFileRequest)}"); return(new ContractExecutionResult()); }