public void DataOceanPath_is_constructed_correctly()
{
const string rootFolder = "rootFolder";
var customerUid = Guid.NewGuid().ToString();
var projectUid = Guid.NewGuid().ToString();
var strResult = DataOceanFileUtil.DataOceanPath(rootFolder, customerUid, projectUid);
Assert.Equal($"{DataOceanUtil.PathSeparator}{rootFolder}{DataOceanUtil.PathSeparator}{customerUid}{DataOceanUtil.PathSeparator}{projectUid}", strResult);
}
/// <summary>
/// Writes the importedFile to DataOcean as a create or update,
/// if it already exists, old version will be deleted first.
/// </summary>
public static async Task WriteFileToDataOcean(
Stream fileContents, string rootFolder, string customerUid, string projectUid, string dataOceanFileName,
ILogger log, IServiceExceptionHandler serviceExceptionHandler, IDataOceanClient dataOceanClient,
ITPaaSApplicationAuthentication authn, Guid fileUid, IConfigurationStore configStore)
{
var dataOceanEnabled = configStore.GetValueBool("ENABLE_DATA_OCEAN", false);
if (dataOceanEnabled)
{
if (!dataOceanFileName.StartsWith(fileUid.ToString()))
{
throw new ServiceException(HttpStatusCode.InternalServerError,
new ContractExecutionResult(ContractExecutionStatesEnum.InternalProcessingError,
$"Invalid DataOcean file name {dataOceanFileName}"));
}
var customHeaders = authn.CustomHeaders();
var dataOceanPath = DataOceanFileUtil.DataOceanPath(rootFolder, customerUid, projectUid);
var ccPutFileResult = false;
var folderAlreadyExists = false;
try
{
log.LogInformation($"{nameof(WriteFileToDataOcean)}: dataOceanPath: '{dataOceanPath}', dataOceanFileName: '{dataOceanFileName}'");
folderAlreadyExists = await dataOceanClient.FolderExists(dataOceanPath, customHeaders);
if (!folderAlreadyExists)
{
await dataOceanClient.MakeFolder(dataOceanPath, customHeaders);
}
ccPutFileResult = await dataOceanClient.PutFile(dataOceanPath, dataOceanFileName, fileContents, customHeaders);
}
catch (Exception e)
{
serviceExceptionHandler.ThrowServiceException(HttpStatusCode.InternalServerError, 57, "dataOceanClient.PutFile",
e.Message);
}
if (!ccPutFileResult)
{
serviceExceptionHandler.ThrowServiceException(HttpStatusCode.InternalServerError, 116);
}
log.LogInformation($"{nameof(WriteFileToDataOcean)}: dataOceanFileName '{dataOceanFileName}' written to DataOcean, folderAlreadyExists: {folderAlreadyExists}");
}
else
{
log.LogInformation($"{nameof(WriteFileToDataOcean)}: File not saved. DataOcean disabled");
}
}
/// <summary>
/// Deletes the importedFile from DataOcean
/// </summary>
public static async Task <ImportedFileInternalResult> DeleteFileFromDataOcean(
string fileName, string rootFolder, string customerUid, Guid projectUid, Guid importedFileUid,
ILogger log, IDataOceanClient dataOceanClient, ITPaaSApplicationAuthentication authn, IConfigurationStore configStore)
{
var dataOceanEnabled = configStore.GetValueBool("ENABLE_DATA_OCEAN", false);
if (dataOceanEnabled)
{
var dataOceanPath = DataOceanFileUtil.DataOceanPath(rootFolder, customerUid, projectUid.ToString());
var fullFileName = $"{dataOceanPath}{Path.DirectorySeparatorChar}{fileName}";
log.LogInformation($"{nameof(DeleteFileFromDataOcean)}: fullFileName {JsonConvert.SerializeObject(fullFileName)}");
var customHeaders = authn.CustomHeaders();
bool ccDeleteFileResult;
try
{
ccDeleteFileResult = await dataOceanClient.DeleteFile(fullFileName, customHeaders);
}
catch (Exception e)
{
log.LogError(e, $"{nameof(DeleteFileFromDataOcean)}: failed for {fileName} (importedFileUid:{importedFileUid}) with exception {e.Message}");
return(ImportedFileInternalResult.CreateImportedFileInternalResult(HttpStatusCode.InternalServerError, 57, "dataOceanClient.DeleteFile", e.Message));
}
if (!ccDeleteFileResult)
{
log.LogWarning(
$"{nameof(DeleteFileFromDataOcean)}: failed to delete {fileName} (importedFileUid:{importedFileUid}).");
//Not an error if it doesn't delete the file?
//return ImportedFileInternalResult.CreateImportedFileInternalResult(HttpStatusCode.InternalServerError, 117);
}
}
else
{
log.LogInformation($"{nameof(DeleteFileFromDataOcean)}: File not deleted. DataOcean disabled");
}
return(null);
}
/// <summary>
/// Deletes file via Trex
/// </summary>
protected override async Task <ContractExecutionResult> ProcessAsyncEx <T>(T item)
{
var deleteImportedFile = CastRequestObjectTo <DeleteImportedFile>(item, errorCode: 68);
await CheckIfUsedInFilter(deleteImportedFile);
await CheckIfHasReferenceSurfacesAsync(deleteImportedFile);
// DB change must be made before productivity3dV2ProxyNotification.DeleteFile is called as it calls back here to get list of Active files
//Don't think the above comment applies any more !
var deleteImportedFileEvent = await ImportedFileRequestDatabaseHelper.DeleteImportedFileInDb
(deleteImportedFile.ProjectUid, deleteImportedFile.ImportedFileUid, serviceExceptionHandler, projectRepo);
ImportedFileInternalResult importedFileInternalResult = null;
if (deleteImportedFile.IsTRexDesignFileType)
{
//Now delete in TRex
await ImportedFileRequestHelper.NotifyTRexDeleteFile(deleteImportedFile.ProjectUid,
deleteImportedFile.ImportedFileType, deleteImportedFile.FileDescriptor.FileName,
deleteImportedFile.ImportedFileUid,
deleteImportedFile.SurveyedUtc,
log, customHeaders, serviceExceptionHandler,
tRexImportFileProxy);
//and from s3 bucket
ProjectRequestHelper.DeleteFileFromS3Repository(
deleteImportedFile.ProjectUid.ToString(), deleteImportedFile.FileDescriptor.FileName,
deleteImportedFile.ImportedFileType == ImportedFileType.SurveyedSurface, deleteImportedFile.SurveyedUtc,
log, serviceExceptionHandler, persistantTransferProxyFactory.NewProxy(TransferProxyType.DesignImport));
}
if (deleteImportedFile.ImportedFileType == ImportedFileType.Linework ||
deleteImportedFile.ImportedFileType == ImportedFileType.GeoTiff)
{
var dataOceanFileName = DataOceanFileUtil.DataOceanFileName(deleteImportedFile.FileDescriptor.FileName,
deleteImportedFile.ImportedFileType == ImportedFileType.SurveyedSurface || deleteImportedFile.ImportedFileType == ImportedFileType.GeoTiff,
deleteImportedFile.ImportedFileUid, deleteImportedFile.SurveyedUtc);
importedFileInternalResult = await DataOceanHelper.DeleteFileFromDataOcean(
dataOceanFileName, deleteImportedFile.DataOceanRootFolder, customerUid,
deleteImportedFile.ProjectUid, deleteImportedFile.ImportedFileUid, log, dataOceanClient, authn, configStore);
var tasks = new List <Task>();
//delete generated DXF tiles
var dxfFileName = DataOceanFileUtil.GeneratedFileName(dataOceanFileName, deleteImportedFile.ImportedFileType);
var dataOceanPath = DataOceanFileUtil.DataOceanPath(deleteImportedFile.DataOceanRootFolder, customerUid, deleteImportedFile.ProjectUid.ToString());
var fullFileName = $"{dataOceanPath}{Path.DirectorySeparatorChar}{dxfFileName}";
tasks.Add(pegasusClient.DeleteTiles(fullFileName, DataOceanHelper.CustomHeaders(authn)));
await Task.WhenAll(tasks);
}
if (importedFileInternalResult != null)
{
await ImportedFileRequestDatabaseHelper.UndeleteImportedFile
(deleteImportedFile.ProjectUid, deleteImportedFile.ImportedFileUid, serviceExceptionHandler, projectRepo);
serviceExceptionHandler.ThrowServiceException(importedFileInternalResult.StatusCode, importedFileInternalResult.ErrorNumber, importedFileInternalResult.ResultCode, importedFileInternalResult.ErrorMessage1);
}
return(new ContractExecutionResult());
}
protected override async Task <ContractExecutionResult> ProcessAsyncEx <T>(T item)
{
List <FileData> files = null;
int zoomLevel = 0;
Point topLeftTile = null;
int numTiles = 0;
BoundingBox2DLatLon bbox = null;
if (item is DxfTileRequest request)
{
files = request.files?.ToList();
bbox = request.bbox;
//Calculate zoom level
zoomLevel = TileServiceUtils.CalculateZoomLevel(request.bbox.TopRightLat - request.bbox.BottomLeftLat,
request.bbox.TopRightLon - request.bbox.BottomLeftLon);
log.LogDebug("DxfTileExecutor: BBOX differences {0} {1} {2}", request.bbox.TopRightLat - request.bbox.BottomLeftLat,
request.bbox.TopRightLon - request.bbox.BottomLeftLon, zoomLevel);
numTiles = TileServiceUtils.NumberOfTiles(zoomLevel);
Point topLeftLatLng = new Point(request.bbox.TopRightLat.LatRadiansToDegrees(),
request.bbox.BottomLeftLon.LonRadiansToDegrees());
topLeftTile = WebMercatorProjection.LatLngToTile(topLeftLatLng, numTiles);
log.LogDebug($"DxfTileExecutor: zoomLevel={zoomLevel}, numTiles={numTiles}, xtile={topLeftTile.x}, ytile={topLeftTile.y}");
}
else if (item is DxfTile3dRequest request3d)
{
files = request3d.files?.ToList();
zoomLevel = request3d.zoomLevel;
numTiles = TileServiceUtils.NumberOfTiles(zoomLevel);
topLeftTile = new Point {
x = request3d.xTile, y = request3d.yTile
};
}
else
{
ThrowRequestTypeCastException <DxfTileRequest>();
}
log.LogDebug($"DxfTileExecutor: {files?.Count ?? 0} files");
//Short circuit overlaying if there no files to overlay as ForAll is an expensive operation
if (files == null || !files.Any())
{
byte[] emptyOverlayData;
using (var bitmap = new Image <Rgba32>(WebMercatorProjection.TILE_SIZE, WebMercatorProjection.TILE_SIZE))
{
emptyOverlayData = bitmap.BitmapToByteArray();
}
return(new TileResult(emptyOverlayData));
}
log.LogDebug(string.Join(",", files.Select(f => f.Name).ToList()));
var tileList = new List <byte[]>();
const string DATA_OCEAN_ROOT_FOLDER_ID_KEY = "DATA_OCEAN_ROOT_FOLDER_ID";
var dataOceanRootFolder = configStore.GetValueString(DATA_OCEAN_ROOT_FOLDER_ID_KEY);
if (string.IsNullOrEmpty(dataOceanRootFolder))
{
throw new ArgumentException($"Missing environment variable {DATA_OCEAN_ROOT_FOLDER_ID_KEY}");
}
//For GeoTIFF files, use the latest version of a file
var geoTiffFiles = files.Where(x => x.ImportedFileType == ImportedFileType.GeoTiff).ToList();
if (geoTiffFiles.Any())
{
//Find any with multiple versions and remove old ones from the list
var latestFiles = geoTiffFiles.GroupBy(g => g.Name).Select(g => g.OrderBy(o => o.SurveyedUtc).Last()).ToList();
foreach (var geoTiffFile in geoTiffFiles)
{
if (!latestFiles.Contains(geoTiffFile))
{
files.Remove(geoTiffFile);
}
}
}
var fileTasks = files.Select(async file =>
{
//foreach (var file in request.files)
//Check file type to see if it has tiles
if (file.ImportedFileType == ImportedFileType.Linework ||
file.ImportedFileType == ImportedFileType.GeoTiff)
{
var fullPath = DataOceanFileUtil.DataOceanPath(dataOceanRootFolder, file.CustomerUid, file.ProjectUid);
var fileName = DataOceanFileUtil.DataOceanFileName(file.Name,
file.ImportedFileType == ImportedFileType.SurveyedSurface || file.ImportedFileType == ImportedFileType.GeoTiff,
Guid.Parse(file.ImportedFileUid), file.SurveyedUtc);
fileName = DataOceanFileUtil.GeneratedFileName(fileName, file.ImportedFileType);
if (zoomLevel >= file.MinZoomLevel)
{
byte[] tileData = null;
if (zoomLevel <= file.MaxZoomLevel || file.MaxZoomLevel == 0) //0 means not calculated
{
tileData = await GetTileAtRequestedZoom(topLeftTile, zoomLevel, fullPath, fileName);
}
else if (zoomLevel - file.MaxZoomLevel <= 5) //Don't try to scale if the difference is too excessive
{
tileData = await GetTileAtHigherZoom(topLeftTile, zoomLevel, fullPath, fileName,
file.MaxZoomLevel, numTiles);
}
else
{
log.LogDebug(
"DxfTileExecutor: difference between requested and maximum zooms too large; not even going to try to scale tile");
}
if (tileData != null && tileData.Length > 0)
{
tileList.Add(tileData);
}
}
}
});
await Task.WhenAll(fileTasks);
log.LogDebug($"DxfTileExecutor: Overlaying {tileList.Count} tiles");
byte[] overlayData = TileOverlay.OverlayTiles(tileList);
return(new TileResult(overlayData));
}