public async Task <ImportedFileDescriptorSingleResult> InternalImportedFileV6(
[FromQuery] string filename,
[FromQuery] string awsFilePath,
[FromQuery] Guid projectUid,
[FromQuery] ImportedFileType importedFileType,
[FromQuery] DxfUnitsType dxfUnitsType,
[FromQuery] DateTime fileCreatedUtc,
[FromQuery] DateTime fileUpdatedUtc,
[FromQuery] DateTime?surveyedUtc,
[FromServices] ITransferProxyFactory transferProxyFactory,
[FromServices] ISchedulerProxy schedulerProxy)
{
if (importedFileType == ImportedFileType.ReferenceSurface)
{
ServiceExceptionHandler.ThrowServiceException(HttpStatusCode.BadRequest, 122);
}
ImportedFileDescriptorSingleResult importedFileResult = null;
var transferProxy = transferProxyFactory.NewProxy(TransferProxyType.Temporary);
Logger.LogInformation(
$"{nameof(InternalImportedFileV6)}:. filename: {filename} awspath {awsFilePath} projectUid {projectUid} ImportedFileType: {importedFileType} " +
$"DxfUnitsType: {dxfUnitsType} surveyedUtc {(surveyedUtc == null ? "N/A" : surveyedUtc.ToString())}");
// Retrieve the stored file from AWS
var fileResult = await transferProxy.Download(awsFilePath);
if (fileResult == null)
{
ServiceExceptionHandler.ThrowServiceException(HttpStatusCode.InternalServerError, 55);
}
using (var ms = new MemoryStream())
{
// Depending on the size of the file in S3, the stream returned may or may not support seeking
// Which we need to TCC to know the length of the file (can't find the length, if you can't seek).
// To solve this, we have to download the entire stream here and copy to memory.
// Allowing TCC to upload the file.
// Not the best solution for extra large files, but TCC doesn't support uploading without file size AFAIK
fileResult.FileStream.CopyTo(ms);
importedFileResult = await UpsertFileInternal(filename, ms, projectUid, importedFileType, dxfUnitsType,
fileCreatedUtc, fileUpdatedUtc, surveyedUtc, schedulerProxy);
}
Logger.LogInformation(
$"{nameof(InternalImportedFileV6)}: Completed successfully. Response: {JsonConvert.SerializeObject(importedFileResult)}");
return(importedFileResult);
}
/// <summary>
/// Adds file via Trex
/// </summary>
protected override async Task <ContractExecutionResult> ProcessAsyncEx <T>(T item)
{
var importedFile = CastRequestObjectTo <CreateImportedFile>(item, errorCode: 68);
await ImportedFileRequestDatabaseHelper.CheckIfParentSurfaceExistsAsync(importedFile.ImportedFileType, importedFile.ParentUid, serviceExceptionHandler, projectRepo);
// need to write to Db prior to
// notifying TRex as Trex needs the ImportedFileUid
var createImportedFileEvent = await ImportedFileRequestDatabaseHelper.CreateImportedFileinDb(
Guid.Parse(customerUid),
importedFile.ProjectUid,
importedFile.ImportedFileType, importedFile.DxfUnitsType, importedFile.FileName,
importedFile.SurveyedUtc, JsonConvert.SerializeObject(importedFile.FileDescriptor),
importedFile.FileCreatedUtc, importedFile.FileUpdatedUtc, userEmailAddress,
log, serviceExceptionHandler, projectRepo, importedFile.ParentUid, importedFile.Offset,
importedFile.ImportedFileUid);
if (importedFile.IsTRexDesignFileType)
{
await ImportedFileRequestHelper.NotifyTRexAddFile(importedFile.ProjectUid,
importedFile.ImportedFileType, importedFile.FileName, createImportedFileEvent.ImportedFileUID,
importedFile.SurveyedUtc,
log, customHeaders, serviceExceptionHandler,
tRexImportFileProxy, projectRepo);
}
if (importedFile.ImportedFileType == ImportedFileType.Linework ||
importedFile.ImportedFileType == ImportedFileType.GeoTiff)
{
var project = ProjectRequestHelper.GetProject(importedFile.ProjectUid, new Guid(customerUid), new Guid(userId), log, serviceExceptionHandler, cwsProjectClient, customHeaders);
var existing = projectRepo.GetImportedFile(createImportedFileEvent.ImportedFileUID.ToString());
await Task.WhenAll(project, existing);
var dcFileName = importedFile.ImportedFileType == ImportedFileType.GeoTiff ?
null :
DataOceanFileUtil.DataOceanFileName(project.Result.CoordinateSystemFileName, false, importedFile.ProjectUid, null);
//Generate raster tiles
var jobRequest = TileGenerationRequestHelper.CreateRequest(
importedFile.ImportedFileType,
customerUid,
importedFile.ProjectUid.ToString(),
existing.Result.ImportedFileUid,
importedFile.DataOceanRootFolder,
importedFile.DataOceanFileName,
dcFileName,
importedFile.DxfUnitsType,
importedFile.SurveyedUtc);
await schedulerProxy.ScheduleVSSJob(jobRequest, customHeaders);
}
var fileDescriptor = new ImportedFileDescriptorSingleResult(
(await ImportedFileRequestDatabaseHelper
.GetImportedFileList(importedFile.ProjectUid.ToString(), log, userId, projectRepo))
.ToImmutableList()
.First(f => f.ImportedFileUid == createImportedFileEvent.ImportedFileUID.ToString())
);
log.LogInformation(
$"CreateImportedFileV4. completed successfully. Response: {JsonConvert.SerializeObject(fileDescriptor)}");
return(fileDescriptor);
}
public ImportFile(string uriRoot = null)
{
this.uriRoot = uriRoot ?? "api/v6/importedfile";
ExpectedImportFileDescriptorSingleResult = new ImportedFileDescriptorSingleResult(ImportFileDescriptor);
}
/// <summary>
/// Common file processing method used by all importedFile endpoints.
/// </summary>
protected async Task <ImportedFileDescriptorSingleResult> UpsertFileInternal(
string filename,
Stream fileStream,
Guid projectUid,
ImportedFileType importedFileType,
DxfUnitsType dxfUnitsType,
DateTime fileCreatedUtc,
DateTime fileUpdatedUtc,
DateTime?surveyedUtc,
ISchedulerProxy schedulerProxy,
Guid?parentUid = null,
double?offset = null)
{
ImportedFileDescriptorSingleResult importedFile = null;
var existing = await ImportedFileRequestDatabaseHelper
.GetImportedFileForProject
(projectUid.ToString(), filename, importedFileType, surveyedUtc,
Logger, ProjectRepo, offset, parentUid)
.ConfigureAwait(false);
var creating = existing == null;
Logger.LogInformation(
creating
? $"{nameof(UpsertFileInternal)}. file doesn't exist already in DB: {filename} projectUid {projectUid} ImportedFileType: {importedFileType} surveyedUtc {(surveyedUtc == null ? "N/A" : surveyedUtc.ToString())} parentUid {parentUid} offset: {offset}"
: $"{nameof(UpsertFileInternal)}. file exists already in DB. Will be updated: {JsonConvert.SerializeObject(existing)}");
FileDescriptor fileDescriptor = null;
var importedFileUid = creating ? Guid.NewGuid() : Guid.Parse(existing.ImportedFileUid);
var dataOceanFileName = DataOceanFileUtil.DataOceanFileName(filename,
importedFileType == ImportedFileType.SurveyedSurface || importedFileType == ImportedFileType.GeoTiff,
importedFileUid, surveyedUtc);
if (importedFileType == ImportedFileType.ReferenceSurface)
{
//FileDescriptor not used for reference surface but validation requires values
fileDescriptor = FileDescriptor.CreateFileDescriptor("Not applicable", "Not applicable", filename);
}
else
{
if (IsTRexDesignFileType(importedFileType))
{
fileDescriptor = ProjectRequestHelper.WriteFileToS3Repository(
fileStream, projectUid.ToString(), filename,
importedFileType == ImportedFileType.SurveyedSurface, surveyedUtc,
Logger, ServiceExceptionHandler, persistantTransferProxyFactory.NewProxy(TransferProxyType.DesignImport));
}
//This is needed for ATs.
fileDescriptor = FileDescriptor.CreateFileDescriptor(
FileSpaceId,
$"/{CustomerUid}/{projectUid}",
filename);
if (importedFileType == ImportedFileType.Linework || importedFileType == ImportedFileType.GeoTiff)
{
//save copy to DataOcean
await DataOceanHelper.WriteFileToDataOcean(
fileStream, DataOceanRootFolderId, CustomerUid, projectUid.ToString(), dataOceanFileName,
Logger, ServiceExceptionHandler, DataOceanClient, Authorization, importedFileUid, ConfigStore);
}
}
if (creating)
{
var createImportedFile = new CreateImportedFile(
projectUid, filename, fileDescriptor, importedFileType, surveyedUtc, dxfUnitsType,
fileCreatedUtc, fileUpdatedUtc, DataOceanRootFolderId, parentUid, offset, importedFileUid, dataOceanFileName);
importedFile = await WithServiceExceptionTryExecuteAsync(() =>
RequestExecutorContainerFactory
.Build <CreateImportedFileExecutor>(
LoggerFactory, ConfigStore, ServiceExceptionHandler, CustomerUid, UserId, UserEmailAddress, customHeaders,
productivity3dV2ProxyCompaction : Productivity3dV2ProxyCompaction,
persistantTransferProxyFactory : persistantTransferProxyFactory, tRexImportFileProxy : tRexImportFileProxy,
projectRepo : ProjectRepo, dataOceanClient : DataOceanClient, authn : Authorization, schedulerProxy : schedulerProxy,
cwsProjectClient : CwsProjectClient)
.ProcessAsync(createImportedFile)
) as ImportedFileDescriptorSingleResult;
Logger.LogInformation(
$"{nameof(UpsertFileInternal)}: Create completed successfully. Response: {JsonConvert.SerializeObject(importedFile)}");
}
else
{
// this also validates that this customer has access to the projectUid
var project = await ProjectRequestHelper.GetProject(projectUid, new Guid(CustomerUid), new Guid(UserId), Logger, ServiceExceptionHandler, CwsProjectClient, customHeaders);
var importedFileUpsertEvent = new UpdateImportedFile(
projectUid, project.ShortRaptorProjectId, importedFileType,
(importedFileType == ImportedFileType.SurveyedSurface || importedFileType == ImportedFileType.GeoTiff)
? surveyedUtc
: null,
dxfUnitsType, fileCreatedUtc, fileUpdatedUtc, fileDescriptor,
Guid.Parse(existing?.ImportedFileUid), existing.ImportedFileId,
DataOceanRootFolderId, offset, dataOceanFileName);
importedFile = await WithServiceExceptionTryExecuteAsync(() =>
RequestExecutorContainerFactory
.Build <UpdateImportedFileExecutor>(
LoggerFactory, ConfigStore, ServiceExceptionHandler, CustomerUid, UserId, UserEmailAddress, customHeaders,
productivity3dV2ProxyCompaction : Productivity3dV2ProxyCompaction,
tRexImportFileProxy : tRexImportFileProxy,
projectRepo : ProjectRepo, dataOceanClient : DataOceanClient, authn : Authorization, schedulerProxy : schedulerProxy,
cwsProjectClient : CwsProjectClient)
.ProcessAsync(importedFileUpsertEvent)
) as ImportedFileDescriptorSingleResult;
Logger.LogInformation(
$"{nameof(UpsertFileInternal)}: Update completed successfully. Response: {JsonConvert.SerializeObject(importedFile)}");
}
await NotificationHubClient.Notify(new ProjectChangedNotification(projectUid));
return(importedFile);
}
/// <summary>
/// Processes the Upsert
/// </summary>
protected override async Task <ContractExecutionResult> ProcessAsyncEx <T>(T item)
{
var importedFile = CastRequestObjectTo <UpdateImportedFile>(item, errorCode: 68);
var existingImportedFile = await projectRepo.GetImportedFile(importedFile.ImportedFileUid.ToString());
if (existingImportedFile == null)
{
throw new ServiceException(HttpStatusCode.BadRequest,
new ContractExecutionResult(ProjectErrorCodesProvider.GetErrorNumberwithOffset(122),
ProjectErrorCodesProvider.FirstNameWithOffset(122)));
}
if (importedFile.IsTRexDesignFileType)
{
await ImportedFileRequestHelper.NotifyTRexUpdateFile(importedFile.ProjectUid,
importedFile.ImportedFileType, importedFile.FileDescriptor.FileName, importedFile.ImportedFileUid,
importedFile.SurveyedUtc,
log, customHeaders, serviceExceptionHandler,
tRexImportFileProxy);
}
if (importedFile.ImportedFileType == ImportedFileType.Linework ||
importedFile.ImportedFileType == ImportedFileType.GeoTiff)
{
string dcFileName = null;
if (importedFile.ImportedFileType == ImportedFileType.Linework)
{
var project = await ProjectRequestHelper.GetProject(importedFile.ProjectUid, new Guid(customerUid), new Guid(userId), log, serviceExceptionHandler, cwsProjectClient, customHeaders);
dcFileName = DataOceanFileUtil.DataOceanFileName(project.CoordinateSystemFileName, false, importedFile.ProjectUid, null);
}
var jobRequest = TileGenerationRequestHelper.CreateRequest(
importedFile.ImportedFileType,
customerUid,
importedFile.ProjectUid.ToString(),
existingImportedFile.ImportedFileUid,
importedFile.DataOceanRootFolder,
importedFile.DataOceanFileName,
dcFileName,
importedFile.DxfUnitsTypeId,
importedFile.SurveyedUtc);
await schedulerProxy.ScheduleVSSJob(jobRequest, customHeaders);
}
// if all succeeds, update Db and put update to kafka que
var updateImportedFileEvent = await ImportedFileRequestDatabaseHelper.UpdateImportedFileInDb(existingImportedFile,
existingImportedFile.FileDescriptor,
importedFile.SurveyedUtc, existingImportedFile.MinZoomLevel, existingImportedFile.MaxZoomLevel,
importedFile.FileCreatedUtc, importedFile.FileUpdatedUtc, userEmailAddress,
log, serviceExceptionHandler, projectRepo);
var fileDescriptor = new ImportedFileDescriptorSingleResult(
(await ImportedFileRequestDatabaseHelper.GetImportedFileList(importedFile.ProjectUid.ToString(), log, userId, projectRepo))
.ToImmutableList()
.FirstOrDefault(f => f.ImportedFileUid == importedFile.ImportedFileUid.ToString())
);
log.LogInformation(
$"UpdateImportedFileExecutor. entry {(fileDescriptor.ImportedFileDescriptor == null ? "not " : "")}retrieved from DB : {JsonConvert.SerializeObject(fileDescriptor)}");
return(fileDescriptor);
}
