/// <summary> /// Loads the SVL design file/s, from storage /// </summary> public override DesignLoadResult LoadFromStorage(Guid siteModelUid, string fileName, string localPath, bool loadIndices = false) { var s3FileTransfer = new S3FileTransfer(TransferProxyType.DesignImport); var isDownloaded = s3FileTransfer.ReadFileSync(siteModelUid, fileName, localPath); return(!isDownloaded ? DesignLoadResult.UnknownFailure : DesignLoadResult.Success); }
private void UploadIndices(S3FileTransfer s3FileTransfer, string s3FileName, string downloadLocalPath, string siteModelUid) { var filename = SpatialIndexFileName(s3FileName); var spatialUploadedOk = s3FileTransfer.WriteFile(downloadLocalPath, Guid.Parse(siteModelUid), filename); if (!spatialUploadedOk) { throw new ArgumentException($"Unable to copy spatial index file to S3: {filename}"); } filename = SubGridIndexFileName(s3FileName); var subgridUploadedOk = s3FileTransfer.WriteFile(downloadLocalPath, Guid.Parse(siteModelUid), filename); if (!subgridUploadedOk) { throw new ArgumentException($"Unable to copy subgrid index file to S3: {filename}"); } // upload boundary... filename = BoundaryFileName(s3FileName); var boundaryUploadedOk = s3FileTransfer.WriteFile(downloadLocalPath, Guid.Parse(siteModelUid), filename); if (!boundaryUploadedOk) { throw new ArgumentException($"Unable to copy boundary file to S3: {filename}"); } }
/// <summary> /// Loads the TTM design file/s, from storage /// Includes design file, 2 index files and a boundary file (if they exist) /// </summary> public override DesignLoadResult LoadFromStorage(Guid siteModelUid, string fileName, string localPath, bool loadIndices = false) { var s3FileTransfer = new S3FileTransfer(TransferProxyType.DesignImport); var isDownloaded = s3FileTransfer.ReadFileSync(siteModelUid, fileName, localPath); if (!isDownloaded) { return(DesignLoadResult.UnknownFailure); } if (loadIndices) { isDownloaded = s3FileTransfer.ReadFileSync(siteModelUid, (fileName + Consts.DESIGN_SUB_GRID_INDEX_FILE_EXTENSION), localPath); if (!isDownloaded) { return(DesignLoadResult.UnableToLoadSubGridIndex); } isDownloaded = s3FileTransfer.ReadFileSync(siteModelUid, (fileName + Consts.DESIGN_SPATIAL_INDEX_FILE_EXTENSION), localPath); if (!isDownloaded) { return(DesignLoadResult.UnableToLoadSpatialIndex); } isDownloaded = s3FileTransfer.ReadFileSync(siteModelUid, (fileName + Consts.DESIGN_BOUNDARY_FILE_EXTENSION), localPath); if (!isDownloaded) { return(DesignLoadResult.UnableToLoadBoundary); } } return(DesignLoadResult.Success); }
public void FileDeleteFromStorage_HappyPath() { var mockTransferProxy = new Mock <ITransferProxy>(); mockTransferProxy.Setup(t => t.RemoveFromBucket(It.IsAny <string>())).Returns(true); var mockTransferProxyFactory = new Mock <ITransferProxyFactory>(); mockTransferProxyFactory.Setup(x => x.NewProxy(It.IsAny <TransferProxyType>())).Returns(mockTransferProxy.Object); var mockConfig = new Mock <IConfigurationStore>(); mockConfig.Setup(x => x.GetValueString("AWS_DESIGNIMPORT_BUCKET_NAME")).Returns("vss-projects-stg"); DIBuilder .New() .AddLogging() .Add(x => x.AddSingleton(mockConfig.Object)) .Add(x => x.AddSingleton(mockTransferProxyFactory.Object)) .Complete(); var projectUid = Guid.Parse("A11F2458-6666-424F-A995-4426a00771AE"); var transferFileName = "TransferTestDesign.ttm"; var s3FileTransfer = new S3FileTransfer(TransferProxyType.DesignImport); var isDeletedFromS3Ok = s3FileTransfer.RemoveFileFromBucket(projectUid, transferFileName); Assert.True(isDeletedFromS3Ok); }
public async Task <CompactionExportResult> PostTINSurface([FromBody] CompactionSurfaceExportRequest compactionSurfaceExportRequest) { Log.LogInformation($"{nameof(PostTINSurface)}: {Request.QueryString}"); Log.LogDebug($"Accept header is {Request.Headers[HeaderConstants.ACCEPT]}"); compactionSurfaceExportRequest.Validate(); ValidateFilterMachines(nameof(PostTINSurface), compactionSurfaceExportRequest.ProjectUid, compactionSurfaceExportRequest.Filter); var tinResult = await WithServiceExceptionTryExecuteAsync(() => RequestExecutorContainer .Build <TINSurfaceExportExecutor>(ConfigStore, LoggerFactory, ServiceExceptionHandler) .ProcessAsync(compactionSurfaceExportRequest)) as TINSurfaceExportResult; if (tinResult?.TINData == null || tinResult?.Code != ContractExecutionStatesEnum.ExecutedSuccessfully) { return(new CompactionExportResult(tinResult.Code, tinResult.Message)); } const string TTM_EXTENSION = ".ttm"; const string ZIP_EXTENSION = ".zip"; var fullFileName = BuildTINFilePath(compactionSurfaceExportRequest.FileName, ZIP_EXTENSION); if (FileSystem.Exists(fullFileName)) { FileSystem.Delete(fullFileName); } using (var zipFile = ZipFile.Open(fullFileName, ZipArchiveMode.Create)) { var entry = zipFile.CreateEntry(compactionSurfaceExportRequest.FileName + TTM_EXTENSION); using (var stream = entry.Open()) new MemoryStream(tinResult?.TINData).CopyTo(stream); } var s3FileTransfer = new S3FileTransfer(TransferProxyType.Temporary); s3FileTransfer.WriteFile(fullFileName, compactionSurfaceExportRequest.ProjectUid, out var url); if (FileSystem.Exists(fullFileName)) // remove temp file { FileSystem.Delete(fullFileName); } return(new CompactionExportResult(url)); }
public override bool RemoveFromStorage(Guid siteModelUid, string fileName) { try { var s3FileTransfer = new S3FileTransfer(TransferProxyType.DesignImport); if (s3FileTransfer.RemoveFileFromBucket(siteModelUid, fileName)) { s3FileTransfer.RemoveFileFromBucket(siteModelUid, fileName + Consts.DESIGN_SUB_GRID_INDEX_FILE_EXTENSION); s3FileTransfer.RemoveFileFromBucket(siteModelUid, fileName + Consts.DESIGN_SPATIAL_INDEX_FILE_EXTENSION); s3FileTransfer.RemoveFileFromBucket(siteModelUid, fileName + Consts.DESIGN_BOUNDARY_FILE_EXTENSION); } } catch (Exception e) { Log.LogError(e, $"Exception RemoveFromStorage. file:{fileName}"); return(false); } return(true); }
public string PersistResult(List <string> dataRows) { bool fileLoadedOk = false; string s3FullPath = null; try { // local path and zip fileName include a unique ID to avoid overwriting someone else file // write file/s to a local, unique, directory var uniqueFileName = requestArgument.FileName + "__" + requestArgument.TRexNodeID; var localExportPath = FilePathHelper.GetTempFolderForExport(requestArgument.ProjectID, uniqueFileName); var localPath = Path.Combine(localExportPath, uniqueFileName); PersistLocally(dataRows, localPath); // zip the directory var zipFullPath = Path.Combine(localExportPath, uniqueFileName) + ZIP_extension; ZipFile.CreateFromDirectory(localPath, zipFullPath, CompressionLevel.Optimal, false, Encoding.UTF8); // copy zip to S3 s3FullPath = $"project/{requestArgument.ProjectID}/TRexExport/{uniqueFileName}{ZIP_extension}"; var s3FileTransfer = new S3FileTransfer(TransferProxyType.Export); fileLoadedOk = s3FileTransfer.WriteFileToBucket(zipFullPath, s3FullPath, awsBucketName); // delete the export folder localExportPath = FilePathHelper.GetTempFolderForExport(requestArgument.ProjectID, ""); if (Directory.Exists(localExportPath) && !retainLocalCopyForTesting) { Directory.Delete(localExportPath, true); } } catch (Exception e) { Log.LogError(e, "Error persisting export data"); throw; } return(fileLoadedOk ? s3FullPath : string.Empty); }
protected override async Task <ContractExecutionResult> ProcessAsyncEx <T>(T item) { var request = CastRequestObjectTo <CompactionCSVExportRequest>(item); var siteModel = GetSiteModel(request.ProjectUid); if (request.CoordType == CoordType.LatLon && siteModel.CSIBLoaded == false) { log.LogError($"#Out# CSVExportExecutor. CoordinateType of LatLong requested, but CSIB not found : Project: {request.ProjectUid} Filename: {request.FileName}"); throw CreateServiceException <CSVExportExecutor> (HttpStatusCode.InternalServerError, ContractExecutionStatesEnum.InternalProcessingError, RequestErrorStatus.ExportInvalidCSIB); } var filter = ConvertFilter(request.Filter, siteModel); var startEndDate = CSVExportHelper.GetDateRange(siteModel, request.Filter); filter.AttributeFilter.StartTime = startEndDate.Item1; filter.AttributeFilter.EndTime = startEndDate.Item2; var tRexRequest = new CSVExportRequest(); var csvExportRequestArgument = AutoMapperUtility.Automapper.Map <CSVExportRequestArgument>(request); csvExportRequestArgument.MappedMachines = CSVExportHelper.MapRequestedMachines(siteModel, request.MachineNames); csvExportRequestArgument.Filters = new FilterSet(filter); var response = await tRexRequest.ExecuteAsync(csvExportRequestArgument); if (response == null || response.ResultStatus != RequestErrorStatus.OK) { log.LogError($"CSVExportExecutor unable to process request. Project: {request.ProjectUid} Filename: {request.FileName} Response: {response?.ResultStatus.ToString()}"); throw CreateServiceException <CSVExportExecutor> (HttpStatusCode.InternalServerError, ContractExecutionStatesEnum.InternalProcessingError, response?.ResultStatus ?? RequestErrorStatus.FailedToConfigureInternalPipeline); } var s3FileTransfer = new S3FileTransfer(TransferProxyType.Temporary); return(new CompactionExportResult(s3FileTransfer.GeneratePreSignedUrl(response.fileName))); }
public override bool RemoveFromStorage(Guid siteModelUid, string fileName) { var s3FileTransfer = new S3FileTransfer(TransferProxyType.DesignImport); return(s3FileTransfer.RemoveFileFromBucket(siteModelUid, fileName)); }
public virtual void SetupFixture() { // At this stage in the DI configuration create a mocked TRexIgniteFactory to permit, for example, storage proxies to // obtain a non-null factory that will return null grid references var mockTRexGridFactory = new Mock <ITRexGridFactory>(); _s3FileTransferProxies = new Dictionary <TransferProxyType, IS3FileTransfer>(); DIBuilder .New() .AddLogging() .Add(VSS.TRex.IO.DIUtilities.AddPoolCachesToDI) .Add(VSS.TRex.Cells.DIUtilities.AddPoolCachesToDI) .Add(x => x.AddSingleton <Mock <IConfigurationStore> >(mock => { var config = new Mock <IConfigurationStore>(); config.Setup(c => c.GetValueInt("NUMPARTITIONS_PERDATACACHE", It.IsAny <int>())).Returns(Consts.NUMPARTITIONS_PERDATACACHE); config.Setup(c => c.GetValueInt("VLPDSUBGRID_SEGMENTPASSCOUNTLIMIT", It.IsAny <int>())).Returns(Consts.VLPDSUBGRID_SEGMENTPASSCOUNTLIMIT); config.Setup(c => c.GetValueInt("VLPDSUBGRID_MAXSEGMENTCELLPASSESLIMIT", It.IsAny <int>())).Returns(Consts.VLPDSUBGRID_MAXSEGMENTCELLPASSESLIMIT); config.Setup(c => c.GetValueBool("SEGMENTCLEAVINGOOPERATIONS_TOLOG", It.IsAny <bool>())).Returns(true /*Consts.SEGMENTCLEAVINGOOPERATIONS_TOLOG*/); config.Setup(c => c.GetValueBool("ITEMSPERSISTEDVIADATAPERSISTOR_TOLOG", It.IsAny <bool>())).Returns(Consts.ITEMSPERSISTEDVIADATAPERSISTOR_TOLOG); config.Setup(c => c.GetValueBool("DEBUG_PERFORMSEGMENT_ADDITIONALINTEGRITYCHECKS", It.IsAny <bool>())).Returns(true /*Consts.DEBUG_PERFORMSEGMENT_ADDITIONALINTEGRITYCHECKS*/); config.Setup(c => c.GetValueInt("VLPDPSNODE_CELL_PASS_AGGREGATOR_LIST_SIZE_INCREMENT_DEFAULT", It.IsAny <int>())).Returns(Consts.VLPDPSNODE_CELL_PASS_AGGREGATOR_LIST_SIZE_INCREMENT_DEFAULT); config.Setup(c => c.GetValueBool("ADVISE_OTHER_SERVICES_OF_MODEL_CHANGES", It.IsAny <bool>())).Returns(true /*Consts.ADVISE_OTHER_SERVICES_OF_MODEL_CHANGES*/); config.Setup(c => c.GetValueInt("MAX_MAPPED_TAG_FILES_TO_PROCESS_PER_AGGREGATION_EPOCH", It.IsAny <int>())).Returns(Consts.MAX_MAPPED_TAG_FILES_TO_PROCESS_PER_AGGREGATION_EPOCH); config.Setup(c => c.GetValueInt("MAX_GROUPED_TAG_FILES_TO_PROCESS_PER_PROCESSING_EPOCH", It.IsAny <int>())).Returns(Consts.MAX_GROUPED_TAG_FILES_TO_PROCESS_PER_PROCESSING_EPOCH); config.Setup(c => c.GetValueInt("HEARTBEAT_LOGGER_INTERVAL")).Returns(Consts.HEARTBEAT_LOGGER_INTERVAL); config.Setup(c => c.GetValueInt("GENERAL_SUBGRID_RESULT_CACHE_MAXIMUM_ELEMENT_COUNT", It.IsAny <int>())).Returns(Consts.GENERAL_SUBGRID_RESULT_CACHE_MAXIMUM_ELEMENT_COUNT); config.Setup(c => c.GetValueLong("GENERAL_SUBGRID_RESULT_CACHE_MAXIMUM_SIZE", It.IsAny <long>())).Returns(Consts.GENERAL_SUBGRID_RESULT_CACHE_MAXIMUM_SIZE); config.Setup(c => c.GetValueDouble("GENERAL_SUBGRID_RESULT_CACHE_DEAD_BAND_FRACTION", It.IsAny <double>())).Returns(Consts.GENERAL_SUBGRID_RESULT_CACHE_DEAD_BAND_FRACTION); config.Setup(c => c.GetValueInt("SUBGRIDTREENODE_CELLSPARCITYLIMIT", It.IsAny <int>())).Returns(Consts.SUBGRIDTREENODE_CELLSPARCITYLIMIT); config.Setup(c => c.GetValueBool("ENABLE_TAGFILE_ARCHIVING_METADATA", It.IsAny <bool>())).Returns(Consts.ENABLE_TAGFILE_ARCHIVING_METADATA); config.Setup(c => c.GetValueBool("ENABLE_TAGFILE_ARCHIVING", It.IsAny <bool>())).Returns(Consts.ENABLE_TAGFILE_ARCHIVING); config.Setup(c => c.GetValueBool("ENABLE_GENERAL_SUBGRID_RESULT_CACHING", It.IsAny <bool>())).Returns(true /*Consts.ENABLE_GENERAL_SUBGRID_RESULT_CACHING*/); config.Setup(c => c.GetValueBool("DEBUG_DRAWDIAGONALCROSS_ONRENDEREDTILES", It.IsAny <bool>())).Returns(true /*Consts.DEBUG_DRAWDIAGONALCROSS_ONRENDEREDTILES*/); config.Setup(c => c.GetValueInt("MAX_EXPORT_ROWS")).Returns(Consts.DEFAULT_MAX_EXPORT_ROWS); config.Setup(c => c.GetValueInt("MAX_EXPORT_ROWS", It.IsAny <int>())).Returns(Consts.DEFAULT_MAX_EXPORT_ROWS); config.Setup(c => c.GetValueInt("SPATIAL_MEMORY_CACHE_INTER_EPOCH_SLEEP_TIME_SECONDS", It.IsAny <int>())).Returns(Consts.SPATIAL_MEMORY_CACHE_INTER_EPOCH_SLEEP_TIME_SECONDS); config.Setup(c => c.GetValueInt("SPATIAL_MEMORY_CACHE_INVALIDATED_CACHE_CONTEXT_REMOVAL_WAIT_TIME_SECONDS", It.IsAny <int>())).Returns(Consts.SPATIAL_MEMORY_CACHE_INVALIDATED_CACHE_CONTEXT_REMOVAL_WAIT_TIME_SECONDS); config.Setup(c => c.GetValueInt("NUM_CONCURRENT_TAG_FILE_PROCESSING_TASKS", It.IsAny <int>())).Returns(Consts.NUM_CONCURRENT_TAG_FILE_PROCESSING_TASKS); config.Setup(c => c.GetValueBool("USE_SYNC_TASKS_FOR_STORAGE_PROXY_IGNITE_TRANSACTIONAL_COMMITS", It.IsAny <bool>())).Returns(true); config.Setup(c => c.GetValueInt("MIN_TAGFILE_LENGTH", It.IsAny <int>())).Returns(Consts.kMinTagFileLengthDefault); config.Setup(c => c.GetValueBool("ENABLE_TFA_SERVICE", It.IsAny <bool>())).Returns(Consts.ENABLE_TFA_SERVICE); config.Setup(c => c.GetValueBool("ENABLE_DEVICE_GATEWAY", It.IsAny <bool>())).Returns(Consts.ENABLE_DEVICE_GATEWAY); config.Setup(c => c.GetValueString("TAGFILE_ARCHIVE_FOLDER", It.IsAny <string>())).Returns(""); config.Setup(c => c.GetValueBool("ENABLE_DEVICE_GATEWAY")).Returns(false); config.Setup(c => c.GetValueBool("ENABLE_DEVICE_GATEWAY", It.IsAny <bool>())).Returns(false); config.Setup(c => c.GetValueString("AWS_TEMPORARY_BUCKET_NAME")).Returns("UnitTestAWSBucketKey"); config.Setup(c => c.GetValueString("AWS_TEMPORARY_BUCKET_NAME", It.IsAny <string>())).Returns("UnitTestAWSBucketKey"); config.Setup(c => c.GetValueBool("SURFACE_EXPORT_DATA_SMOOTHING_ACTIVE", It.IsAny <bool>())).Returns(false); config.Setup(c => c.GetValueInt("SURFACE_EXPORT_DATA_SMOOTHING_NULL_INFILL_MODE", It.IsAny <int>())).Returns((int)NullInfillMode.NoInfill); config.Setup(c => c.GetValueInt("SURFACE_EXPORT_DATA_SMOOTHING_MASK_SIZE", It.IsAny <int>())).Returns((int)ConvolutionMaskSize.Mask3X3); config.Setup(c => c.GetValueBool("TILE_RENDERING_DATA_SMOOTHING_ACTIVE", It.IsAny <bool>())).Returns(false); config.Setup(c => c.GetValueInt("TILE_RENDERING_DATA_SMOOTHING_NULL_INFILL_MODE", It.IsAny <int>())).Returns((int)NullInfillMode.NoInfill); config.Setup(c => c.GetValueInt("TILE_RENDERING_DATA_SMOOTHING_MASK_SIZE", It.IsAny <int>())).Returns((int)ConvolutionMaskSize.Mask3X3); config.Setup(c => c.GetValueInt("SUB_GRIDS_REQUEST_ADDRESS_BUCKET_SIZE", It.IsAny <int>())).Returns(50); var tempPersistencePathForTests = Path.GetTempPath(); config.Setup(c => c.GetValueString("PERSISTENT_CACHE_STORE_LOCATION", It.IsAny <string>())).Returns(tempPersistencePathForTests); config.Setup(c => c.GetValueString("PERSISTENT_CACHE_STORE_LOCATION")).Returns(tempPersistencePathForTests); config.Setup(c => c.GetValueBool("USE_LOCAL_S3_TRANSFER_PROXY_STORE", It.IsAny <bool>())).Returns(true); config.Setup(c => c.GetValueBool("USE_LOCAL_S3_TRANSFER_PROXY_STORE")).Returns(true); config.Setup(c => c.GetValueString("AWS_TAGFILE_BUCKET_NAME", It.IsAny <string>())).Returns("AWS_TAGFILE_BUCKET"); config.Setup(c => c.GetValueString("AWS_TAGFILE_BUCKET_NAME")).Returns("AWS_TAGFILE_BUCKET"); config.Setup(c => c.GetValueString("AWS_DESIGNIMPORT_BUCKET_NAME", It.IsAny <string>())).Returns("AWS_DESIGNIMPORT_BUCKET"); config.Setup(c => c.GetValueString("AWS_DESIGNIMPORT_BUCKET_NAME")).Returns("AWS_DESIGNIMPORT_BUCKET"); config.Setup(c => c.GetValueInt("REBUILD_SITE_MODEL_MONITORING_INTERVAL_MS")).Returns(1000); config.Setup(c => c.GetValueInt("REBUILD_SITE_MODEL_MONITORING_INTERVAL_MS", It.IsAny <int>())).Returns(1000); config.Setup(c => c.GetValueString("TGL_GEODATA_PATH", It.IsAny <string>())).Returns("Geodata"); config.Setup(c => c.GetValueUlong("TREX_DESIGN_ELEVATION_CACHE_SIZE", It.IsAny <ulong>())).Returns((ulong)10 * 1024 * 1024); config.Setup(c => c.GetValueInt("TREX_QOS_SCHEDULER_DEFAULT_THREAD_POOL_FRACTION_DIVISOR")).Returns(4); config.Setup(c => c.GetValueInt("TREX_QOS_SCHEDULER_DEFAULT_THREAD_POOL_FRACTION_DIVISOR", It.IsAny <int>())).Returns(4); config.Setup(c => c.GetValueInt("TREX_QOS_SCHEDULER_TASK_GROUP_TIMEOUT_SECONDS")).Returns(10); config.Setup(c => c.GetValueInt("TREX_QOS_SCHEDULER_TASK_GROUP_TIMEOUT_SECONDS", It.IsAny <int>())).Returns(10); config.Setup(c => c.GetValueInt("TREX_QOS_SCHEDULER_MAX_CONCURRENT_SCHEDULER_SESSIONS")).Returns(4); config.Setup(c => c.GetValueInt("TREX_QOS_SCHEDULER_MAX_CONCURRENT_SCHEDULER_SESSIONS", It.IsAny <int>())).Returns(4); return(config); })) .Build() .Add(x => x.AddSingleton <IConfigurationStore>(DIContext.Obtain <Mock <IConfigurationStore> >().Object)) .Add(x => x.AddSingleton(ClientLeafSubGridFactoryFactory.CreateClientSubGridFactory())) .Add(x => x.AddSingleton <ISubGridCellSegmentPassesDataWrapperFactory>(new SubGridCellSegmentPassesDataWrapperFactory())) .Add(x => x.AddSingleton <ISubGridCellLatestPassesDataWrapperFactory>(new SubGridCellLatestPassesDataWrapperFactory())) .Add(x => x.AddSingleton <ISubGridSpatialAffinityKeyFactory>(new SubGridSpatialAffinityKeyFactory())) .Add(x => x.AddSingleton <ITransferProxyFactory, TransferProxyFactory>()) .Add(x => x.AddSingleton <Func <TransferProxyType, IS3FileTransfer> > (factory => proxyType => { if (_s3FileTransferProxies.TryGetValue(proxyType, out var proxy)) { return(proxy); } proxy = new S3FileTransfer(proxyType); _s3FileTransferProxies.Add(proxyType, proxy); return(proxy); })) .Add(x => x.AddSingleton(mockTRexGridFactory.Object)) .Complete(); }
/// <summary> /// Add a design or surveyed surface or alignment file to TRex /// </summary> protected async Task AddDesign(DesignRequest request, string executorName) { // load core file from s3 to local var localPath = FilePathHelper.GetTempFolderForProject(request.ProjectUid); var localPathAndFileName = Path.Combine(new[] { localPath, request.FileName }); DesignBase design; if (request.FileType == ImportedFileType.Alignment) { design = new SVLAlignmentDesign(); } else { design = new TTMDesign(SubGridTreeConsts.DefaultCellSize); } var designLoadResult = design.LoadFromStorage(request.ProjectUid, request.FileName, localPath); if (designLoadResult != DesignLoadResult.Success) { log.LogError($"#Out# {executorName}. Addition of design failed :{request.FileName}, Project:{request.ProjectUid}, DesignUid:{request.DesignUid}, designLoadResult: {designLoadResult}"); throw CreateServiceException <TExecutor> (HttpStatusCode.InternalServerError, ContractExecutionStatesEnum.InternalProcessingError, RequestErrorStatus.DesignImportUnableToRetrieveFromS3, designLoadResult.ToString()); } if (request.FileType != ImportedFileType.Alignment) { // This generates the 2 index files designLoadResult = design.LoadFromFile(localPathAndFileName); if (designLoadResult != DesignLoadResult.Success) { log.LogError($"#Out# {executorName}. Addition of design failed :{request.FileName}, Project:{request.ProjectUid}, DesignUid:{request.DesignUid}, designLoadResult: {designLoadResult}"); throw CreateServiceException <TExecutor> (HttpStatusCode.InternalServerError, ContractExecutionStatesEnum.InternalProcessingError, RequestErrorStatus.DesignImportUnableToCreateDesign, designLoadResult.ToString()); } } var extents = new BoundingWorldExtent3D(); design.GetExtents(out extents.MinX, out extents.MinY, out extents.MaxX, out extents.MaxY); design.GetHeightRange(out extents.MinZ, out extents.MaxZ); if (request.FileType == ImportedFileType.DesignSurface) { // Create the new designSurface in our site var tRexRequest = new AddTTMDesignRequest(); var designSurfaceUid = await tRexRequest.ExecuteAsync(new AddTTMDesignArgument { ProjectID = request.ProjectUid, DesignDescriptor = new Designs.Models.DesignDescriptor(request.DesignUid, localPathAndFileName, request.FileName), Extents = extents, ExistenceMap = design.SubGridOverlayIndex() }); } if (request.FileType == ImportedFileType.SurveyedSurface) { // Create the new SurveyedSurface in our site model var tRexRequest = new AddSurveyedSurfaceRequest(); var surveyedSurfaceUid = await tRexRequest.ExecuteAsync(new AddSurveyedSurfaceArgument { ProjectID = request.ProjectUid, DesignDescriptor = new Designs.Models.DesignDescriptor(request.DesignUid, localPathAndFileName, request.FileName), AsAtDate = request.SurveyedUtc ?? TRex.Common.Consts.MIN_DATETIME_AS_UTC, // validation will have ensured this exists Extents = extents, ExistenceMap = design.SubGridOverlayIndex() }); } if (request.FileType == ImportedFileType.Alignment) { // Create the new alignment in our site model var tRexRequest = new AddAlignmentRequest(); var alignmentUid = await tRexRequest.ExecuteAsync(new AddAlignmentArgument { ProjectID = request.ProjectUid, DesignDescriptor = new Designs.Models.DesignDescriptor(request.DesignUid, localPathAndFileName, request.FileName), Extents = extents }); } if (request.FileType != ImportedFileType.Alignment) { // TTM.LoadFromFile() will have created these 3 files. We need to store them on S3 to reload cache when required var s3FileTransfer = new S3FileTransfer(TransferProxyType.DesignImport); s3FileTransfer.WriteFile(localPath, request.ProjectUid, SubGridIndexFileName(request.FileName)); s3FileTransfer.WriteFile(localPath, request.ProjectUid, SpatialIndexFileName(request.FileName)); s3FileTransfer.WriteFile(localPath, request.ProjectUid, BoundaryFileName(request.FileName)); } }
/// <summary> /// Delete a design or surveyed surface or alignment file from TRex /// </summary> protected async Task RemoveDesign(DesignRequest request, string executorName) { bool removedOk = false; if (request.FileType == ImportedFileType.DesignSurface) { // Remove the designSurface var tRexRequest = new RemoveTTMDesignRequest(); var removeResponse = await tRexRequest.ExecuteAsync(new RemoveTTMDesignArgument { ProjectID = request.ProjectUid, DesignID = request.DesignUid }); removedOk = removeResponse.RequestResult == DesignProfilerRequestResult.OK; } if (request.FileType == ImportedFileType.SurveyedSurface) { // Remove the new surveyedSurface var tRexRequest = new RemoveSurveyedSurfaceRequest(); var removeResponse = await tRexRequest.ExecuteAsync(new RemoveSurveyedSurfaceArgument { ProjectID = request.ProjectUid, DesignID = request.DesignUid }); removedOk = removeResponse.RequestResult == DesignProfilerRequestResult.OK; } if (request.FileType == ImportedFileType.Alignment) { // Remove the alignment var tRexRequest = new RemoveAlignmentRequest(); var removeResponse = await tRexRequest.ExecuteAsync(new RemoveAlignmentArgument { ProjectID = request.ProjectUid, AlignmentID = request.DesignUid }); removedOk = removeResponse.RequestResult == DesignProfilerRequestResult.OK; } if (!removedOk) { log.LogError($"#Out# {executorName}. Deletion failed, of design:{request.FileName}, Project:{request.ProjectUid}, DesignUid:{request.DesignUid}"); throw CreateServiceException <TExecutor> (HttpStatusCode.InternalServerError, ContractExecutionStatesEnum.InternalProcessingError, RequestErrorStatus.DesignImportUnableToDeleteDesign); } //Remove local copies of files var localPath = FilePathHelper.GetTempFolderForProject(request.ProjectUid); var localPathAndFileName = Path.Combine(new[] { localPath, request.FileName }); if (File.Exists(localPathAndFileName)) { try { File.Delete(localPathAndFileName); if (request.FileType != ImportedFileType.Alignment) { //Delete index files var indexFileName = SubGridIndexFileName(localPathAndFileName); if (File.Exists(indexFileName)) { File.Delete(indexFileName); } indexFileName = SpatialIndexFileName(localPathAndFileName); if (File.Exists(indexFileName)) { File.Delete(indexFileName); } indexFileName = BoundaryFileName(localPathAndFileName); if (File.Exists(indexFileName)) { File.Delete(indexFileName); } } } catch (Exception e) { log.LogError(e, $"Failed to delete files related to design/surveyed surface {request.DesignUid} in project {request.ProjectUid}"); } } if (request.FileType != ImportedFileType.Alignment) { //Remove the index files from s3 (project service removes the actual file from s3 as it put it there originally) var s3FileTransfer = new S3FileTransfer(TransferProxyType.DesignImport); s3FileTransfer.RemoveFileFromBucket(request.ProjectUid, SubGridIndexFileName(request.FileName)); s3FileTransfer.RemoveFileFromBucket(request.ProjectUid, SpatialIndexFileName(request.FileName)); s3FileTransfer.RemoveFileFromBucket(request.ProjectUid, BoundaryFileName(request.FileName)); } }
public async Task <JsonResult> AddDesignToSiteModel( string siteModelUid, string importedFileType, [FromQuery] string fileNameAndLocalPath, [FromQuery] string asAtDate, [FromQuery] Guid designUid) { var importedFileTypeEnum = ValidateImportedFileType(importedFileType); if (string.IsNullOrEmpty(siteModelUid)) { throw new ArgumentException($"Invalid siteModelUid (you need to have selected one first): {siteModelUid}"); } if (string.IsNullOrEmpty(fileNameAndLocalPath) || !Path.HasExtension(fileNameAndLocalPath) || (importedFileTypeEnum != ImportedFileType.Alignment && (string.Compare(Path.GetExtension(fileNameAndLocalPath), ".ttm", StringComparison.OrdinalIgnoreCase) != 0)) || (importedFileTypeEnum == ImportedFileType.Alignment && (string.Compare(Path.GetExtension(fileNameAndLocalPath), ".svl", StringComparison.OrdinalIgnoreCase) != 0)) ) { throw new ArgumentException($"Invalid [path]filename: {fileNameAndLocalPath}"); } if (!System.IO.File.Exists(fileNameAndLocalPath)) { throw new ArgumentException($"Unable to locate [path]fileName: {fileNameAndLocalPath}"); } var siteModelGuid = Guid.Parse(siteModelUid); if (designUid == Guid.Empty) { designUid = Guid.NewGuid(); } // copy local file to S3 bool designFileLoadedOk; string s3FileName; var destinationFileName = string.Empty; var tempFileNameOnly = Path.GetFileName(fileNameAndLocalPath); var s3FileTransfer = new S3FileTransfer(TransferProxyType.DesignImport); if (importedFileTypeEnum == ImportedFileType.SurveyedSurface) { var tempDate = asAtDate.Remove(asAtDate.IndexOf(".", 0, StringComparison.Ordinal)).Replace(":", ""); destinationFileName = Path.GetFileNameWithoutExtension(tempFileNameOnly) + $"_{tempDate}" + Path.GetExtension(tempFileNameOnly); designFileLoadedOk = s3FileTransfer.WriteFile(Path.GetDirectoryName(fileNameAndLocalPath), Guid.Parse(siteModelUid), tempFileNameOnly, destinationFileName); s3FileName = destinationFileName; } else { designFileLoadedOk = s3FileTransfer.WriteFile(Path.GetDirectoryName(fileNameAndLocalPath), Guid.Parse(siteModelUid), tempFileNameOnly); s3FileName = tempFileNameOnly; } if (!designFileLoadedOk) { throw new ArgumentException($"Unable to copy design file to S3: {s3FileName}"); } // download to appropriate local location and add to site model var downloadLocalPath = FilePathHelper.GetTempFolderForProject(Guid.Parse(siteModelUid)); var downloadedok = await s3FileTransfer.ReadFile(Guid.Parse(siteModelUid), s3FileName, downloadLocalPath).ConfigureAwait(false); if (!downloadedok) { throw new ArgumentException($"Unable to restore same design file from S3: {s3FileName}"); } if (importedFileTypeEnum == ImportedFileType.DesignSurface) { AddTheDesignSurfaceToSiteModel(siteModelGuid, designUid, downloadLocalPath, s3FileName); // upload indices UploadIndices(s3FileTransfer, s3FileName, downloadLocalPath, siteModelUid); return(new JsonResult(DIContext.Obtain <IDesignManager>().List(siteModelGuid).Locate(designUid))); } if (importedFileTypeEnum == ImportedFileType.SurveyedSurface) { var surveyedUtc = DateTime.Parse(asAtDate).ToUniversalTime(); //DateTime.UtcNow; // unable to parse the date from UI DateTime.Parse(asAtDate); AddTheSurveyedSurfaceToSiteModel(siteModelGuid, designUid, downloadLocalPath, s3FileName, surveyedUtc); // upload indices UploadIndices(s3FileTransfer, s3FileName, downloadLocalPath, siteModelUid); return(new JsonResult(DIContext.Obtain <ISurveyedSurfaceManager>().List(siteModelGuid).Locate(designUid))); } if (importedFileTypeEnum == ImportedFileType.Alignment) { AddTheAlignmentToSiteModel(siteModelGuid, designUid, downloadLocalPath, s3FileName); return(new JsonResult(DIContext.Obtain <IAlignmentManager>().List(siteModelGuid).Locate(designUid))); } throw new ArgumentException($"{nameof(AddDesignToSiteModel)} Unsupported ImportedFileType: {importedFileType}"); }