/// <summary>
/// Loads the SVL design file/s, from storage
/// </summary>
public override DesignLoadResult LoadFromStorage(Guid siteModelUid, string fileName, string localPath, bool loadIndices = false)
{
var s3FileTransfer = new S3FileTransfer(TransferProxyType.DesignImport);
var isDownloaded = s3FileTransfer.ReadFileSync(siteModelUid, fileName, localPath);
return(!isDownloaded ? DesignLoadResult.UnknownFailure : DesignLoadResult.Success);
}
private void UploadIndices(S3FileTransfer s3FileTransfer, string s3FileName, string downloadLocalPath, string siteModelUid)
{
var filename = SpatialIndexFileName(s3FileName);
var spatialUploadedOk = s3FileTransfer.WriteFile(downloadLocalPath, Guid.Parse(siteModelUid), filename);
if (!spatialUploadedOk)
{
throw new ArgumentException($"Unable to copy spatial index file to S3: {filename}");
}
filename = SubGridIndexFileName(s3FileName);
var subgridUploadedOk = s3FileTransfer.WriteFile(downloadLocalPath, Guid.Parse(siteModelUid), filename);
if (!subgridUploadedOk)
{
throw new ArgumentException($"Unable to copy subgrid index file to S3: {filename}");
}
// upload boundary...
filename = BoundaryFileName(s3FileName);
var boundaryUploadedOk = s3FileTransfer.WriteFile(downloadLocalPath, Guid.Parse(siteModelUid), filename);
if (!boundaryUploadedOk)
{
throw new ArgumentException($"Unable to copy boundary file to S3: {filename}");
}
}
/// <summary>
/// Loads the TTM design file/s, from storage
/// Includes design file, 2 index files and a boundary file (if they exist)
/// </summary>
public override DesignLoadResult LoadFromStorage(Guid siteModelUid, string fileName, string localPath, bool loadIndices = false)
{
var s3FileTransfer = new S3FileTransfer(TransferProxyType.DesignImport);
var isDownloaded = s3FileTransfer.ReadFileSync(siteModelUid, fileName, localPath);
if (!isDownloaded)
{
return(DesignLoadResult.UnknownFailure);
}
if (loadIndices)
{
isDownloaded = s3FileTransfer.ReadFileSync(siteModelUid, (fileName + Consts.DESIGN_SUB_GRID_INDEX_FILE_EXTENSION), localPath);
if (!isDownloaded)
{
return(DesignLoadResult.UnableToLoadSubGridIndex);
}
isDownloaded = s3FileTransfer.ReadFileSync(siteModelUid, (fileName + Consts.DESIGN_SPATIAL_INDEX_FILE_EXTENSION), localPath);
if (!isDownloaded)
{
return(DesignLoadResult.UnableToLoadSpatialIndex);
}
isDownloaded = s3FileTransfer.ReadFileSync(siteModelUid, (fileName + Consts.DESIGN_BOUNDARY_FILE_EXTENSION), localPath);
if (!isDownloaded)
{
return(DesignLoadResult.UnableToLoadBoundary);
}
}
return(DesignLoadResult.Success);
}
public void FileDeleteFromStorage_HappyPath()
{
var mockTransferProxy = new Mock <ITransferProxy>();
mockTransferProxy.Setup(t => t.RemoveFromBucket(It.IsAny <string>())).Returns(true);
var mockTransferProxyFactory = new Mock <ITransferProxyFactory>();
mockTransferProxyFactory.Setup(x => x.NewProxy(It.IsAny <TransferProxyType>())).Returns(mockTransferProxy.Object);
var mockConfig = new Mock <IConfigurationStore>();
mockConfig.Setup(x => x.GetValueString("AWS_DESIGNIMPORT_BUCKET_NAME")).Returns("vss-projects-stg");
DIBuilder
.New()
.AddLogging()
.Add(x => x.AddSingleton(mockConfig.Object))
.Add(x => x.AddSingleton(mockTransferProxyFactory.Object))
.Complete();
var projectUid = Guid.Parse("A11F2458-6666-424F-A995-4426a00771AE");
var transferFileName = "TransferTestDesign.ttm";
var s3FileTransfer = new S3FileTransfer(TransferProxyType.DesignImport);
var isDeletedFromS3Ok = s3FileTransfer.RemoveFileFromBucket(projectUid, transferFileName);
Assert.True(isDeletedFromS3Ok);
}
public async Task <CompactionExportResult> PostTINSurface([FromBody] CompactionSurfaceExportRequest compactionSurfaceExportRequest)
{
Log.LogInformation($"{nameof(PostTINSurface)}: {Request.QueryString}");
Log.LogDebug($"Accept header is {Request.Headers[HeaderConstants.ACCEPT]}");
compactionSurfaceExportRequest.Validate();
ValidateFilterMachines(nameof(PostTINSurface), compactionSurfaceExportRequest.ProjectUid, compactionSurfaceExportRequest.Filter);
var tinResult = await WithServiceExceptionTryExecuteAsync(() =>
RequestExecutorContainer
.Build <TINSurfaceExportExecutor>(ConfigStore, LoggerFactory, ServiceExceptionHandler)
.ProcessAsync(compactionSurfaceExportRequest)) as TINSurfaceExportResult;
if (tinResult?.TINData == null || tinResult?.Code != ContractExecutionStatesEnum.ExecutedSuccessfully)
{
return(new CompactionExportResult(tinResult.Code, tinResult.Message));
}
const string TTM_EXTENSION = ".ttm";
const string ZIP_EXTENSION = ".zip";
var fullFileName = BuildTINFilePath(compactionSurfaceExportRequest.FileName, ZIP_EXTENSION);
if (FileSystem.Exists(fullFileName))
{
FileSystem.Delete(fullFileName);
}
using (var zipFile = ZipFile.Open(fullFileName, ZipArchiveMode.Create))
{
var entry = zipFile.CreateEntry(compactionSurfaceExportRequest.FileName + TTM_EXTENSION);
using (var stream = entry.Open())
new MemoryStream(tinResult?.TINData).CopyTo(stream);
}
var s3FileTransfer = new S3FileTransfer(TransferProxyType.Temporary);
s3FileTransfer.WriteFile(fullFileName, compactionSurfaceExportRequest.ProjectUid, out var url);
if (FileSystem.Exists(fullFileName)) // remove temp file
{
FileSystem.Delete(fullFileName);
}
return(new CompactionExportResult(url));
}
public override bool RemoveFromStorage(Guid siteModelUid, string fileName)
{
try
{
var s3FileTransfer = new S3FileTransfer(TransferProxyType.DesignImport);
if (s3FileTransfer.RemoveFileFromBucket(siteModelUid, fileName))
{
s3FileTransfer.RemoveFileFromBucket(siteModelUid, fileName + Consts.DESIGN_SUB_GRID_INDEX_FILE_EXTENSION);
s3FileTransfer.RemoveFileFromBucket(siteModelUid, fileName + Consts.DESIGN_SPATIAL_INDEX_FILE_EXTENSION);
s3FileTransfer.RemoveFileFromBucket(siteModelUid, fileName + Consts.DESIGN_BOUNDARY_FILE_EXTENSION);
}
}
catch (Exception e)
{
Log.LogError(e, $"Exception RemoveFromStorage. file:{fileName}");
return(false);
}
return(true);
}
public string PersistResult(List <string> dataRows)
{
bool fileLoadedOk = false;
string s3FullPath = null;
try
{
// local path and zip fileName include a unique ID to avoid overwriting someone else file
// write file/s to a local, unique, directory
var uniqueFileName = requestArgument.FileName + "__" + requestArgument.TRexNodeID;
var localExportPath = FilePathHelper.GetTempFolderForExport(requestArgument.ProjectID, uniqueFileName);
var localPath = Path.Combine(localExportPath, uniqueFileName);
PersistLocally(dataRows, localPath);
// zip the directory
var zipFullPath = Path.Combine(localExportPath, uniqueFileName) + ZIP_extension;
ZipFile.CreateFromDirectory(localPath, zipFullPath, CompressionLevel.Optimal, false, Encoding.UTF8);
// copy zip to S3
s3FullPath = $"project/{requestArgument.ProjectID}/TRexExport/{uniqueFileName}{ZIP_extension}";
var s3FileTransfer = new S3FileTransfer(TransferProxyType.Export);
fileLoadedOk = s3FileTransfer.WriteFileToBucket(zipFullPath, s3FullPath, awsBucketName);
// delete the export folder
localExportPath = FilePathHelper.GetTempFolderForExport(requestArgument.ProjectID, "");
if (Directory.Exists(localExportPath) && !retainLocalCopyForTesting)
{
Directory.Delete(localExportPath, true);
}
}
catch (Exception e)
{
Log.LogError(e, "Error persisting export data");
throw;
}
return(fileLoadedOk ? s3FullPath : string.Empty);
}
protected override async Task <ContractExecutionResult> ProcessAsyncEx <T>(T item)
{
var request = CastRequestObjectTo <CompactionCSVExportRequest>(item);
var siteModel = GetSiteModel(request.ProjectUid);
if (request.CoordType == CoordType.LatLon && siteModel.CSIBLoaded == false)
{
log.LogError($"#Out# CSVExportExecutor. CoordinateType of LatLong requested, but CSIB not found : Project: {request.ProjectUid} Filename: {request.FileName}");
throw CreateServiceException <CSVExportExecutor>
(HttpStatusCode.InternalServerError, ContractExecutionStatesEnum.InternalProcessingError,
RequestErrorStatus.ExportInvalidCSIB);
}
var filter = ConvertFilter(request.Filter, siteModel);
var startEndDate = CSVExportHelper.GetDateRange(siteModel, request.Filter);
filter.AttributeFilter.StartTime = startEndDate.Item1;
filter.AttributeFilter.EndTime = startEndDate.Item2;
var tRexRequest = new CSVExportRequest();
var csvExportRequestArgument = AutoMapperUtility.Automapper.Map <CSVExportRequestArgument>(request);
csvExportRequestArgument.MappedMachines = CSVExportHelper.MapRequestedMachines(siteModel, request.MachineNames);
csvExportRequestArgument.Filters = new FilterSet(filter);
var response = await tRexRequest.ExecuteAsync(csvExportRequestArgument);
if (response == null || response.ResultStatus != RequestErrorStatus.OK)
{
log.LogError($"CSVExportExecutor unable to process request. Project: {request.ProjectUid} Filename: {request.FileName} Response: {response?.ResultStatus.ToString()}");
throw CreateServiceException <CSVExportExecutor>
(HttpStatusCode.InternalServerError, ContractExecutionStatesEnum.InternalProcessingError,
response?.ResultStatus ?? RequestErrorStatus.FailedToConfigureInternalPipeline);
}
var s3FileTransfer = new S3FileTransfer(TransferProxyType.Temporary);
return(new CompactionExportResult(s3FileTransfer.GeneratePreSignedUrl(response.fileName)));
}
public override bool RemoveFromStorage(Guid siteModelUid, string fileName)
{
var s3FileTransfer = new S3FileTransfer(TransferProxyType.DesignImport);
return(s3FileTransfer.RemoveFileFromBucket(siteModelUid, fileName));
}
public virtual void SetupFixture()
{
// At this stage in the DI configuration create a mocked TRexIgniteFactory to permit, for example, storage proxies to
// obtain a non-null factory that will return null grid references
var mockTRexGridFactory = new Mock <ITRexGridFactory>();
_s3FileTransferProxies = new Dictionary <TransferProxyType, IS3FileTransfer>();
DIBuilder
.New()
.AddLogging()
.Add(VSS.TRex.IO.DIUtilities.AddPoolCachesToDI)
.Add(VSS.TRex.Cells.DIUtilities.AddPoolCachesToDI)
.Add(x => x.AddSingleton <Mock <IConfigurationStore> >(mock =>
{
var config = new Mock <IConfigurationStore>();
config.Setup(c => c.GetValueInt("NUMPARTITIONS_PERDATACACHE", It.IsAny <int>())).Returns(Consts.NUMPARTITIONS_PERDATACACHE);
config.Setup(c => c.GetValueInt("VLPDSUBGRID_SEGMENTPASSCOUNTLIMIT", It.IsAny <int>())).Returns(Consts.VLPDSUBGRID_SEGMENTPASSCOUNTLIMIT);
config.Setup(c => c.GetValueInt("VLPDSUBGRID_MAXSEGMENTCELLPASSESLIMIT", It.IsAny <int>())).Returns(Consts.VLPDSUBGRID_MAXSEGMENTCELLPASSESLIMIT);
config.Setup(c => c.GetValueBool("SEGMENTCLEAVINGOOPERATIONS_TOLOG", It.IsAny <bool>())).Returns(true /*Consts.SEGMENTCLEAVINGOOPERATIONS_TOLOG*/);
config.Setup(c => c.GetValueBool("ITEMSPERSISTEDVIADATAPERSISTOR_TOLOG", It.IsAny <bool>())).Returns(Consts.ITEMSPERSISTEDVIADATAPERSISTOR_TOLOG);
config.Setup(c => c.GetValueBool("DEBUG_PERFORMSEGMENT_ADDITIONALINTEGRITYCHECKS", It.IsAny <bool>())).Returns(true /*Consts.DEBUG_PERFORMSEGMENT_ADDITIONALINTEGRITYCHECKS*/);
config.Setup(c => c.GetValueInt("VLPDPSNODE_CELL_PASS_AGGREGATOR_LIST_SIZE_INCREMENT_DEFAULT", It.IsAny <int>())).Returns(Consts.VLPDPSNODE_CELL_PASS_AGGREGATOR_LIST_SIZE_INCREMENT_DEFAULT);
config.Setup(c => c.GetValueBool("ADVISE_OTHER_SERVICES_OF_MODEL_CHANGES", It.IsAny <bool>())).Returns(true /*Consts.ADVISE_OTHER_SERVICES_OF_MODEL_CHANGES*/);
config.Setup(c => c.GetValueInt("MAX_MAPPED_TAG_FILES_TO_PROCESS_PER_AGGREGATION_EPOCH", It.IsAny <int>())).Returns(Consts.MAX_MAPPED_TAG_FILES_TO_PROCESS_PER_AGGREGATION_EPOCH);
config.Setup(c => c.GetValueInt("MAX_GROUPED_TAG_FILES_TO_PROCESS_PER_PROCESSING_EPOCH", It.IsAny <int>())).Returns(Consts.MAX_GROUPED_TAG_FILES_TO_PROCESS_PER_PROCESSING_EPOCH);
config.Setup(c => c.GetValueInt("HEARTBEAT_LOGGER_INTERVAL")).Returns(Consts.HEARTBEAT_LOGGER_INTERVAL);
config.Setup(c => c.GetValueInt("GENERAL_SUBGRID_RESULT_CACHE_MAXIMUM_ELEMENT_COUNT", It.IsAny <int>())).Returns(Consts.GENERAL_SUBGRID_RESULT_CACHE_MAXIMUM_ELEMENT_COUNT);
config.Setup(c => c.GetValueLong("GENERAL_SUBGRID_RESULT_CACHE_MAXIMUM_SIZE", It.IsAny <long>())).Returns(Consts.GENERAL_SUBGRID_RESULT_CACHE_MAXIMUM_SIZE);
config.Setup(c => c.GetValueDouble("GENERAL_SUBGRID_RESULT_CACHE_DEAD_BAND_FRACTION", It.IsAny <double>())).Returns(Consts.GENERAL_SUBGRID_RESULT_CACHE_DEAD_BAND_FRACTION);
config.Setup(c => c.GetValueInt("SUBGRIDTREENODE_CELLSPARCITYLIMIT", It.IsAny <int>())).Returns(Consts.SUBGRIDTREENODE_CELLSPARCITYLIMIT);
config.Setup(c => c.GetValueBool("ENABLE_TAGFILE_ARCHIVING_METADATA", It.IsAny <bool>())).Returns(Consts.ENABLE_TAGFILE_ARCHIVING_METADATA);
config.Setup(c => c.GetValueBool("ENABLE_TAGFILE_ARCHIVING", It.IsAny <bool>())).Returns(Consts.ENABLE_TAGFILE_ARCHIVING);
config.Setup(c => c.GetValueBool("ENABLE_GENERAL_SUBGRID_RESULT_CACHING", It.IsAny <bool>())).Returns(true /*Consts.ENABLE_GENERAL_SUBGRID_RESULT_CACHING*/);
config.Setup(c => c.GetValueBool("DEBUG_DRAWDIAGONALCROSS_ONRENDEREDTILES", It.IsAny <bool>())).Returns(true /*Consts.DEBUG_DRAWDIAGONALCROSS_ONRENDEREDTILES*/);
config.Setup(c => c.GetValueInt("MAX_EXPORT_ROWS")).Returns(Consts.DEFAULT_MAX_EXPORT_ROWS);
config.Setup(c => c.GetValueInt("MAX_EXPORT_ROWS", It.IsAny <int>())).Returns(Consts.DEFAULT_MAX_EXPORT_ROWS);
config.Setup(c => c.GetValueInt("SPATIAL_MEMORY_CACHE_INTER_EPOCH_SLEEP_TIME_SECONDS", It.IsAny <int>())).Returns(Consts.SPATIAL_MEMORY_CACHE_INTER_EPOCH_SLEEP_TIME_SECONDS);
config.Setup(c => c.GetValueInt("SPATIAL_MEMORY_CACHE_INVALIDATED_CACHE_CONTEXT_REMOVAL_WAIT_TIME_SECONDS", It.IsAny <int>())).Returns(Consts.SPATIAL_MEMORY_CACHE_INVALIDATED_CACHE_CONTEXT_REMOVAL_WAIT_TIME_SECONDS);
config.Setup(c => c.GetValueInt("NUM_CONCURRENT_TAG_FILE_PROCESSING_TASKS", It.IsAny <int>())).Returns(Consts.NUM_CONCURRENT_TAG_FILE_PROCESSING_TASKS);
config.Setup(c => c.GetValueBool("USE_SYNC_TASKS_FOR_STORAGE_PROXY_IGNITE_TRANSACTIONAL_COMMITS", It.IsAny <bool>())).Returns(true);
config.Setup(c => c.GetValueInt("MIN_TAGFILE_LENGTH", It.IsAny <int>())).Returns(Consts.kMinTagFileLengthDefault);
config.Setup(c => c.GetValueBool("ENABLE_TFA_SERVICE", It.IsAny <bool>())).Returns(Consts.ENABLE_TFA_SERVICE);
config.Setup(c => c.GetValueBool("ENABLE_DEVICE_GATEWAY", It.IsAny <bool>())).Returns(Consts.ENABLE_DEVICE_GATEWAY);
config.Setup(c => c.GetValueString("TAGFILE_ARCHIVE_FOLDER", It.IsAny <string>())).Returns("");
config.Setup(c => c.GetValueBool("ENABLE_DEVICE_GATEWAY")).Returns(false);
config.Setup(c => c.GetValueBool("ENABLE_DEVICE_GATEWAY", It.IsAny <bool>())).Returns(false);
config.Setup(c => c.GetValueString("AWS_TEMPORARY_BUCKET_NAME")).Returns("UnitTestAWSBucketKey");
config.Setup(c => c.GetValueString("AWS_TEMPORARY_BUCKET_NAME", It.IsAny <string>())).Returns("UnitTestAWSBucketKey");
config.Setup(c => c.GetValueBool("SURFACE_EXPORT_DATA_SMOOTHING_ACTIVE", It.IsAny <bool>())).Returns(false);
config.Setup(c => c.GetValueInt("SURFACE_EXPORT_DATA_SMOOTHING_NULL_INFILL_MODE", It.IsAny <int>())).Returns((int)NullInfillMode.NoInfill);
config.Setup(c => c.GetValueInt("SURFACE_EXPORT_DATA_SMOOTHING_MASK_SIZE", It.IsAny <int>())).Returns((int)ConvolutionMaskSize.Mask3X3);
config.Setup(c => c.GetValueBool("TILE_RENDERING_DATA_SMOOTHING_ACTIVE", It.IsAny <bool>())).Returns(false);
config.Setup(c => c.GetValueInt("TILE_RENDERING_DATA_SMOOTHING_NULL_INFILL_MODE", It.IsAny <int>())).Returns((int)NullInfillMode.NoInfill);
config.Setup(c => c.GetValueInt("TILE_RENDERING_DATA_SMOOTHING_MASK_SIZE", It.IsAny <int>())).Returns((int)ConvolutionMaskSize.Mask3X3);
config.Setup(c => c.GetValueInt("SUB_GRIDS_REQUEST_ADDRESS_BUCKET_SIZE", It.IsAny <int>())).Returns(50);
var tempPersistencePathForTests = Path.GetTempPath();
config.Setup(c => c.GetValueString("PERSISTENT_CACHE_STORE_LOCATION", It.IsAny <string>())).Returns(tempPersistencePathForTests);
config.Setup(c => c.GetValueString("PERSISTENT_CACHE_STORE_LOCATION")).Returns(tempPersistencePathForTests);
config.Setup(c => c.GetValueBool("USE_LOCAL_S3_TRANSFER_PROXY_STORE", It.IsAny <bool>())).Returns(true);
config.Setup(c => c.GetValueBool("USE_LOCAL_S3_TRANSFER_PROXY_STORE")).Returns(true);
config.Setup(c => c.GetValueString("AWS_TAGFILE_BUCKET_NAME", It.IsAny <string>())).Returns("AWS_TAGFILE_BUCKET");
config.Setup(c => c.GetValueString("AWS_TAGFILE_BUCKET_NAME")).Returns("AWS_TAGFILE_BUCKET");
config.Setup(c => c.GetValueString("AWS_DESIGNIMPORT_BUCKET_NAME", It.IsAny <string>())).Returns("AWS_DESIGNIMPORT_BUCKET");
config.Setup(c => c.GetValueString("AWS_DESIGNIMPORT_BUCKET_NAME")).Returns("AWS_DESIGNIMPORT_BUCKET");
config.Setup(c => c.GetValueInt("REBUILD_SITE_MODEL_MONITORING_INTERVAL_MS")).Returns(1000);
config.Setup(c => c.GetValueInt("REBUILD_SITE_MODEL_MONITORING_INTERVAL_MS", It.IsAny <int>())).Returns(1000);
config.Setup(c => c.GetValueString("TGL_GEODATA_PATH", It.IsAny <string>())).Returns("Geodata");
config.Setup(c => c.GetValueUlong("TREX_DESIGN_ELEVATION_CACHE_SIZE", It.IsAny <ulong>())).Returns((ulong)10 * 1024 * 1024);
config.Setup(c => c.GetValueInt("TREX_QOS_SCHEDULER_DEFAULT_THREAD_POOL_FRACTION_DIVISOR")).Returns(4);
config.Setup(c => c.GetValueInt("TREX_QOS_SCHEDULER_DEFAULT_THREAD_POOL_FRACTION_DIVISOR", It.IsAny <int>())).Returns(4);
config.Setup(c => c.GetValueInt("TREX_QOS_SCHEDULER_TASK_GROUP_TIMEOUT_SECONDS")).Returns(10);
config.Setup(c => c.GetValueInt("TREX_QOS_SCHEDULER_TASK_GROUP_TIMEOUT_SECONDS", It.IsAny <int>())).Returns(10);
config.Setup(c => c.GetValueInt("TREX_QOS_SCHEDULER_MAX_CONCURRENT_SCHEDULER_SESSIONS")).Returns(4);
config.Setup(c => c.GetValueInt("TREX_QOS_SCHEDULER_MAX_CONCURRENT_SCHEDULER_SESSIONS", It.IsAny <int>())).Returns(4);
return(config);
}))
.Build()
.Add(x => x.AddSingleton <IConfigurationStore>(DIContext.Obtain <Mock <IConfigurationStore> >().Object))
.Add(x => x.AddSingleton(ClientLeafSubGridFactoryFactory.CreateClientSubGridFactory()))
.Add(x => x.AddSingleton <ISubGridCellSegmentPassesDataWrapperFactory>(new SubGridCellSegmentPassesDataWrapperFactory()))
.Add(x => x.AddSingleton <ISubGridCellLatestPassesDataWrapperFactory>(new SubGridCellLatestPassesDataWrapperFactory()))
.Add(x => x.AddSingleton <ISubGridSpatialAffinityKeyFactory>(new SubGridSpatialAffinityKeyFactory()))
.Add(x => x.AddSingleton <ITransferProxyFactory, TransferProxyFactory>())
.Add(x => x.AddSingleton <Func <TransferProxyType, IS3FileTransfer> >
(factory => proxyType =>
{
if (_s3FileTransferProxies.TryGetValue(proxyType, out var proxy))
{
return(proxy);
}
proxy = new S3FileTransfer(proxyType);
_s3FileTransferProxies.Add(proxyType, proxy);
return(proxy);
}))
.Add(x => x.AddSingleton(mockTRexGridFactory.Object))
.Complete();
}
/// <summary>
/// Add a design or surveyed surface or alignment file to TRex
/// </summary>
protected async Task AddDesign(DesignRequest request, string executorName)
{
// load core file from s3 to local
var localPath = FilePathHelper.GetTempFolderForProject(request.ProjectUid);
var localPathAndFileName = Path.Combine(new[] { localPath, request.FileName });
DesignBase design;
if (request.FileType == ImportedFileType.Alignment)
{
design = new SVLAlignmentDesign();
}
else
{
design = new TTMDesign(SubGridTreeConsts.DefaultCellSize);
}
var designLoadResult = design.LoadFromStorage(request.ProjectUid, request.FileName, localPath);
if (designLoadResult != DesignLoadResult.Success)
{
log.LogError($"#Out# {executorName}. Addition of design failed :{request.FileName}, Project:{request.ProjectUid}, DesignUid:{request.DesignUid}, designLoadResult: {designLoadResult}");
throw CreateServiceException <TExecutor>
(HttpStatusCode.InternalServerError, ContractExecutionStatesEnum.InternalProcessingError,
RequestErrorStatus.DesignImportUnableToRetrieveFromS3, designLoadResult.ToString());
}
if (request.FileType != ImportedFileType.Alignment)
{
// This generates the 2 index files
designLoadResult = design.LoadFromFile(localPathAndFileName);
if (designLoadResult != DesignLoadResult.Success)
{
log.LogError($"#Out# {executorName}. Addition of design failed :{request.FileName}, Project:{request.ProjectUid}, DesignUid:{request.DesignUid}, designLoadResult: {designLoadResult}");
throw CreateServiceException <TExecutor>
(HttpStatusCode.InternalServerError, ContractExecutionStatesEnum.InternalProcessingError,
RequestErrorStatus.DesignImportUnableToCreateDesign, designLoadResult.ToString());
}
}
var extents = new BoundingWorldExtent3D();
design.GetExtents(out extents.MinX, out extents.MinY, out extents.MaxX, out extents.MaxY);
design.GetHeightRange(out extents.MinZ, out extents.MaxZ);
if (request.FileType == ImportedFileType.DesignSurface)
{
// Create the new designSurface in our site
var tRexRequest = new AddTTMDesignRequest();
var designSurfaceUid = await tRexRequest.ExecuteAsync(new AddTTMDesignArgument
{
ProjectID = request.ProjectUid,
DesignDescriptor = new Designs.Models.DesignDescriptor(request.DesignUid, localPathAndFileName, request.FileName),
Extents = extents,
ExistenceMap = design.SubGridOverlayIndex()
});
}
if (request.FileType == ImportedFileType.SurveyedSurface)
{
// Create the new SurveyedSurface in our site model
var tRexRequest = new AddSurveyedSurfaceRequest();
var surveyedSurfaceUid = await tRexRequest.ExecuteAsync(new AddSurveyedSurfaceArgument
{
ProjectID = request.ProjectUid,
DesignDescriptor = new Designs.Models.DesignDescriptor(request.DesignUid, localPathAndFileName, request.FileName),
AsAtDate = request.SurveyedUtc ?? TRex.Common.Consts.MIN_DATETIME_AS_UTC, // validation will have ensured this exists
Extents = extents,
ExistenceMap = design.SubGridOverlayIndex()
});
}
if (request.FileType == ImportedFileType.Alignment)
{
// Create the new alignment in our site model
var tRexRequest = new AddAlignmentRequest();
var alignmentUid = await tRexRequest.ExecuteAsync(new AddAlignmentArgument
{
ProjectID = request.ProjectUid,
DesignDescriptor = new Designs.Models.DesignDescriptor(request.DesignUid, localPathAndFileName, request.FileName),
Extents = extents
});
}
if (request.FileType != ImportedFileType.Alignment)
{
// TTM.LoadFromFile() will have created these 3 files. We need to store them on S3 to reload cache when required
var s3FileTransfer = new S3FileTransfer(TransferProxyType.DesignImport);
s3FileTransfer.WriteFile(localPath, request.ProjectUid, SubGridIndexFileName(request.FileName));
s3FileTransfer.WriteFile(localPath, request.ProjectUid, SpatialIndexFileName(request.FileName));
s3FileTransfer.WriteFile(localPath, request.ProjectUid, BoundaryFileName(request.FileName));
}
}
/// <summary>
/// Delete a design or surveyed surface or alignment file from TRex
/// </summary>
protected async Task RemoveDesign(DesignRequest request, string executorName)
{
bool removedOk = false;
if (request.FileType == ImportedFileType.DesignSurface)
{
// Remove the designSurface
var tRexRequest = new RemoveTTMDesignRequest();
var removeResponse = await tRexRequest.ExecuteAsync(new RemoveTTMDesignArgument
{
ProjectID = request.ProjectUid,
DesignID = request.DesignUid
});
removedOk = removeResponse.RequestResult == DesignProfilerRequestResult.OK;
}
if (request.FileType == ImportedFileType.SurveyedSurface)
{
// Remove the new surveyedSurface
var tRexRequest = new RemoveSurveyedSurfaceRequest();
var removeResponse = await tRexRequest.ExecuteAsync(new RemoveSurveyedSurfaceArgument
{
ProjectID = request.ProjectUid,
DesignID = request.DesignUid
});
removedOk = removeResponse.RequestResult == DesignProfilerRequestResult.OK;
}
if (request.FileType == ImportedFileType.Alignment)
{
// Remove the alignment
var tRexRequest = new RemoveAlignmentRequest();
var removeResponse = await tRexRequest.ExecuteAsync(new RemoveAlignmentArgument
{
ProjectID = request.ProjectUid,
AlignmentID = request.DesignUid
});
removedOk = removeResponse.RequestResult == DesignProfilerRequestResult.OK;
}
if (!removedOk)
{
log.LogError($"#Out# {executorName}. Deletion failed, of design:{request.FileName}, Project:{request.ProjectUid}, DesignUid:{request.DesignUid}");
throw CreateServiceException <TExecutor>
(HttpStatusCode.InternalServerError, ContractExecutionStatesEnum.InternalProcessingError,
RequestErrorStatus.DesignImportUnableToDeleteDesign);
}
//Remove local copies of files
var localPath = FilePathHelper.GetTempFolderForProject(request.ProjectUid);
var localPathAndFileName = Path.Combine(new[] { localPath, request.FileName });
if (File.Exists(localPathAndFileName))
{
try
{
File.Delete(localPathAndFileName);
if (request.FileType != ImportedFileType.Alignment)
{
//Delete index files
var indexFileName = SubGridIndexFileName(localPathAndFileName);
if (File.Exists(indexFileName))
{
File.Delete(indexFileName);
}
indexFileName = SpatialIndexFileName(localPathAndFileName);
if (File.Exists(indexFileName))
{
File.Delete(indexFileName);
}
indexFileName = BoundaryFileName(localPathAndFileName);
if (File.Exists(indexFileName))
{
File.Delete(indexFileName);
}
}
}
catch (Exception e)
{
log.LogError(e, $"Failed to delete files related to design/surveyed surface {request.DesignUid} in project {request.ProjectUid}");
}
}
if (request.FileType != ImportedFileType.Alignment)
{
//Remove the index files from s3 (project service removes the actual file from s3 as it put it there originally)
var s3FileTransfer = new S3FileTransfer(TransferProxyType.DesignImport);
s3FileTransfer.RemoveFileFromBucket(request.ProjectUid, SubGridIndexFileName(request.FileName));
s3FileTransfer.RemoveFileFromBucket(request.ProjectUid, SpatialIndexFileName(request.FileName));
s3FileTransfer.RemoveFileFromBucket(request.ProjectUid, BoundaryFileName(request.FileName));
}
}
public async Task <JsonResult> AddDesignToSiteModel(
string siteModelUid,
string importedFileType,
[FromQuery] string fileNameAndLocalPath,
[FromQuery] string asAtDate,
[FromQuery] Guid designUid)
{
var importedFileTypeEnum = ValidateImportedFileType(importedFileType);
if (string.IsNullOrEmpty(siteModelUid))
{
throw new ArgumentException($"Invalid siteModelUid (you need to have selected one first): {siteModelUid}");
}
if (string.IsNullOrEmpty(fileNameAndLocalPath) ||
!Path.HasExtension(fileNameAndLocalPath) ||
(importedFileTypeEnum != ImportedFileType.Alignment &&
(string.Compare(Path.GetExtension(fileNameAndLocalPath), ".ttm", StringComparison.OrdinalIgnoreCase) != 0))
||
(importedFileTypeEnum == ImportedFileType.Alignment &&
(string.Compare(Path.GetExtension(fileNameAndLocalPath), ".svl", StringComparison.OrdinalIgnoreCase) != 0))
)
{
throw new ArgumentException($"Invalid [path]filename: {fileNameAndLocalPath}");
}
if (!System.IO.File.Exists(fileNameAndLocalPath))
{
throw new ArgumentException($"Unable to locate [path]fileName: {fileNameAndLocalPath}");
}
var siteModelGuid = Guid.Parse(siteModelUid);
if (designUid == Guid.Empty)
{
designUid = Guid.NewGuid();
}
// copy local file to S3
bool designFileLoadedOk;
string s3FileName;
var destinationFileName = string.Empty;
var tempFileNameOnly = Path.GetFileName(fileNameAndLocalPath);
var s3FileTransfer = new S3FileTransfer(TransferProxyType.DesignImport);
if (importedFileTypeEnum == ImportedFileType.SurveyedSurface)
{
var tempDate = asAtDate.Remove(asAtDate.IndexOf(".", 0, StringComparison.Ordinal)).Replace(":", "");
destinationFileName = Path.GetFileNameWithoutExtension(tempFileNameOnly) + $"_{tempDate}" + Path.GetExtension(tempFileNameOnly);
designFileLoadedOk = s3FileTransfer.WriteFile(Path.GetDirectoryName(fileNameAndLocalPath), Guid.Parse(siteModelUid), tempFileNameOnly, destinationFileName);
s3FileName = destinationFileName;
}
else
{
designFileLoadedOk = s3FileTransfer.WriteFile(Path.GetDirectoryName(fileNameAndLocalPath), Guid.Parse(siteModelUid), tempFileNameOnly);
s3FileName = tempFileNameOnly;
}
if (!designFileLoadedOk)
{
throw new ArgumentException($"Unable to copy design file to S3: {s3FileName}");
}
// download to appropriate local location and add to site model
var downloadLocalPath = FilePathHelper.GetTempFolderForProject(Guid.Parse(siteModelUid));
var downloadedok = await s3FileTransfer.ReadFile(Guid.Parse(siteModelUid), s3FileName, downloadLocalPath).ConfigureAwait(false);
if (!downloadedok)
{
throw new ArgumentException($"Unable to restore same design file from S3: {s3FileName}");
}
if (importedFileTypeEnum == ImportedFileType.DesignSurface)
{
AddTheDesignSurfaceToSiteModel(siteModelGuid, designUid, downloadLocalPath, s3FileName);
// upload indices
UploadIndices(s3FileTransfer, s3FileName, downloadLocalPath, siteModelUid);
return(new JsonResult(DIContext.Obtain <IDesignManager>().List(siteModelGuid).Locate(designUid)));
}
if (importedFileTypeEnum == ImportedFileType.SurveyedSurface)
{
var surveyedUtc = DateTime.Parse(asAtDate).ToUniversalTime(); //DateTime.UtcNow; // unable to parse the date from UI DateTime.Parse(asAtDate);
AddTheSurveyedSurfaceToSiteModel(siteModelGuid, designUid, downloadLocalPath, s3FileName, surveyedUtc);
// upload indices
UploadIndices(s3FileTransfer, s3FileName, downloadLocalPath, siteModelUid);
return(new JsonResult(DIContext.Obtain <ISurveyedSurfaceManager>().List(siteModelGuid).Locate(designUid)));
}
if (importedFileTypeEnum == ImportedFileType.Alignment)
{
AddTheAlignmentToSiteModel(siteModelGuid, designUid, downloadLocalPath, s3FileName);
return(new JsonResult(DIContext.Obtain <IAlignmentManager>().List(siteModelGuid).Locate(designUid)));
}
throw new ArgumentException($"{nameof(AddDesignToSiteModel)} Unsupported ImportedFileType: {importedFileType}");
}
