public FileDataSingleResult CreateMockImportedFile(
FlowFile file,
[FromQuery] Guid projectUid,
[FromQuery] ImportedFileType importedFileType,
[FromQuery] DxfUnitsType dxfUnitsType,
[FromQuery] DateTime fileCreatedUtc,
[FromQuery] DateTime fileUpdatedUtc,
[FromQuery] DateTime?surveyedUtc = null)
{
Logger.LogInformation(
$"CreateMockImportedFile. file: {file.flowFilename} path {file.path} projectUid {projectUid} ImportedFileType: {importedFileType} " +
$"DxfUnitsType: {dxfUnitsType} surveyedUtc {(surveyedUtc == null ? "N/A" : surveyedUtc.ToString())}");
if (projectUid.ToString() == ConstantsUtil.DIMENSIONS_PROJECT_UID)
{
var result = new FileDataSingleResult();
result.ImportedFileDescriptor = ImportedFilesService.ImportedFiles[ConstantsUtil.DIMENSIONS_PROJECT_UID]
.SingleOrDefault(f => f.Name.Equals(file.flowFilename, StringComparison.OrdinalIgnoreCase));
return(result);
}
return(new FileDataSingleResult
{
Code = ContractExecutionStatesEnum.InternalProcessingError,
Message = "Failed to create imported file"
});
}
public async Task <ImportedFileDescriptorSingleResult> CreateImportedFileDirectV6(
[FromServices] ISchedulerProxy schedulerProxy,
Guid projectUid,
string filename,
ImportedFileType importedFileType,
DxfUnitsType dxfUnitsType,
DateTime fileCreatedUtc,
DateTime fileUpdatedUtc,
DateTime?surveyedUtc = null)
{
if (importedFileType == ImportedFileType.ReferenceSurface)
{
ServiceExceptionHandler.ThrowServiceException(HttpStatusCode.BadRequest, 122);
}
FileImportDataValidator.ValidateUpsertImportedFileRequest(projectUid, importedFileType, dxfUnitsType, fileCreatedUtc,
fileUpdatedUtc, UserEmailAddress, surveyedUtc, filename, null, null);
Logger.LogInformation(
$"{nameof(CreateImportedFileDirectV6)}: ProjectUID: `{projectUid}`, Filename: `{filename}` ImportedFileType: `{importedFileType}`, DxfUnitsType: `{dxfUnitsType}`, SurveyedUTC: `{(surveyedUtc == null ? "N/A" : surveyedUtc.ToString())}`");
//When debugging locally using Postman, remove this check so can do an update
await ValidateFileDoesNotExist(projectUid.ToString(), filename, importedFileType, surveyedUtc, null, null);
if (!MultipartRequestHelper.IsMultipartContentType(Request.ContentType))
{
ServiceExceptionHandler.ThrowServiceException(HttpStatusCode.BadRequest, 58, $"Expected a multipart request, but got '{Request.ContentType}'");
}
var tempFilePath = await HttpContext.Request.StreamFile(Guid.NewGuid().ToString(), Logger);
var result = await UpsertFile(tempFilePath, filename, projectUid.ToString(), importedFileType, dxfUnitsType, fileCreatedUtc, fileUpdatedUtc, surveyedUtc, schedulerProxy);
return(result);
}
public DXFUtilitiesResult Extract(DxfUnitsType units, uint maxBoundariesToProcess, bool convertLineStringCoordsToPolygon)
{
bool atEOF;
Boundaries = new PolyLineBoundaries(units, maxBoundariesToProcess);
do
{
if (_reader.GetBoundaryFromPolyLineEntity(!convertLineStringCoordsToPolygon, out atEOF, out var boundary))
{
if (boundary != null && boundary.Boundary.Points.Count >= 3)
{
if (boundary.Type == DXFLineWorkBoundaryType.Unknown)
{
boundary.Type = DXFLineWorkBoundaryType.GenericBoundary;
}
if (string.IsNullOrEmpty(boundary.Name))
{
boundary.Name = $"{Boundaries.Boundaries.Count + 1}";
}
Boundaries.Boundaries.Add(boundary);
}
}
} while (Boundaries.Boundaries.Count < maxBoundariesToProcess && !atEOF);
// Determine units in file
return(DXFUtilitiesResult.Ok);
}
/// <summary>
/// Create instance of ProjectFileDescriptor using ID
/// </summary>
public static ProjectFileDescriptor CreateProjectFileDescriptor
(
long?projectId,
Guid?projectUId,
FileDescriptor file,
string coordSystemFileName,
DxfUnitsType dxfUnitsType,
long fileId,
ImportedFileType fileType,
Guid fileUid,
string userEmailAddress,
long?legacyFileId = null
)
{
return(new ProjectFileDescriptor
{
ProjectId = projectId,
ProjectUid = projectUId,
File = file,
CoordSystemFileName = coordSystemFileName,
DXFUnitsType = dxfUnitsType,
FileId = fileId,
FileType = fileType,
FileUid = fileUid,
UserEmailAddress = userEmailAddress,
LegacyFileId = legacyFileId
});
}
/// <summary>
/// Construct a DXFReader given a stream reader representing teh DXF file content and the units to use
/// </summary>
/// <param name="dxfFile"></param>
/// <param name="units"></param>
public DXFReader(MemoryStream dxfFile, DxfUnitsType units)
{
DXFFileIsBinary = IsBinaryDXF(dxfFile, out _dXFFileIsPostR13c3);
if (DXFFileIsBinary)
{
_binarydxfFile = dxfFile;
_binarydxfFile.Position = BinaryACADDXFSignature.Length; // Skip the sentinel
}
else
{
dxfFile.Position = 0;
_aSCIIdxfFile = new StreamReader(dxfFile);
}
Units = units;
DXFImportConvFactor = units switch
{
DxfUnitsType.Meters => UnitUtils.DistToMeters(DistanceUnitsType.Meters),
DxfUnitsType.UsSurveyFeet => UnitUtils.DistToMeters(DistanceUnitsType.US_feet),
DxfUnitsType.ImperialFeet => UnitUtils.DistToMeters(DistanceUnitsType.Feet),
_ => 1.0
};
}
}
private async void TestAFile(string fileName, DxfUnitsType units, int expectedBoundaryCount, int firstBoundaryVertexCount, string expectedName, DXFLineWorkBoundaryType expectedType, bool allowUnclosedBoundaries)
{
var request = new DXFBoundariesRequest("", ImportedFileType.SiteBoundary,
Convert.ToBase64String(File.ReadAllBytes(Path.Combine("TestData", fileName))),
units, 10, allowUnclosedBoundaries);
var executor = new ExtractDXFBoundariesExecutor(DIContext.Obtain <IConfigurationStore>(), DIContext.Obtain <ILoggerFactory>(), DIContext.Obtain <IServiceExceptionHandler>());
executor.Should().NotBeNull();
var result = await executor.ProcessAsync(request);
result.Should().NotBeNull();
result.Code.Should().Be(ContractExecutionStatesEnum.ExecutedSuccessfully);
result.Message.Should().Be("Success");
if (result is DXFBoundaryResult boundary)
{
boundary.Boundaries.Count.Should().Be(expectedBoundaryCount);
if (expectedBoundaryCount > 0)
{
boundary.Boundaries[0].Fence.Count.Should().Be(firstBoundaryVertexCount);
boundary.Boundaries[0].Name.Should().Be(expectedName);
boundary.Boundaries[0].Type.Should().Be(expectedType);
}
}
else
{
false.Should().BeTrue(); // fail the test
}
}
public async void Boundaries_OverLimit(string fileName, DxfUnitsType units, int firstBoundaryVertexCount)
{
const int LIMIT = 5;
var request = new DXFBoundariesRequest("", ImportedFileType.SiteBoundary,
Convert.ToBase64String(File.ReadAllBytes(Path.Combine("TestData", fileName))), units, LIMIT, false);
var executor = new ExtractDXFBoundariesExecutor(DIContext.Obtain <IConfigurationStore>(), DIContext.Obtain <ILoggerFactory>(), DIContext.Obtain <IServiceExceptionHandler>());
executor.Should().NotBeNull();
var result = await executor.ProcessAsync(request);
result.Should().NotBeNull();
result.Code.Should().Be(ContractExecutionStatesEnum.ExecutedSuccessfully);
result.Message.Should().Be("Success");
if (result is DXFBoundaryResult boundary)
{
boundary.Boundaries.Count.Should().Be(LIMIT);
boundary.Boundaries[0].Fence.Count.Should().Be(firstBoundaryVertexCount);
}
else
{
false.Should().BeTrue(); // fail the test
}
}
public async Task <ScheduleJobResult> BackgroundUpload(
FlowFile file,
[FromQuery] Guid projectUid,
[FromQuery] ImportedFileType importedFileType,
[FromQuery] DxfUnitsType dxfUnitsType,
[FromQuery] DateTime fileCreatedUtc,
[FromQuery] DateTime fileUpdatedUtc,
[FromQuery] DateTime?surveyedUtc,
[FromServices] ISchedulerProxy scheduler,
[FromServices] ITransferProxyFactory transferProxyFactory)
{
if (importedFileType == ImportedFileType.ReferenceSurface)
{
ServiceExceptionHandler.ThrowServiceException(HttpStatusCode.BadRequest, 122);
}
FlowJsFileImportDataValidator.ValidateUpsertImportedFileRequest(
file, projectUid, importedFileType, dxfUnitsType, fileCreatedUtc, fileUpdatedUtc, UserEmailAddress, surveyedUtc, null, null);
Logger.LogInformation(
$"{nameof(BackgroundUpload)}: file: {file.flowFilename} path {file.path} projectUid {projectUid} ImportedFileType: {importedFileType} " +
$"DxfUnitsType: {dxfUnitsType} surveyedUtc {(surveyedUtc == null ? "N/A" : surveyedUtc.ToString())}");
if (string.Equals(Request.Method, HttpMethod.Post.ToString(), StringComparison.OrdinalIgnoreCase))
{
await ValidateFileDoesNotExist(projectUid.ToString(), file.flowFilename, importedFileType, surveyedUtc, null, null);
}
var s3Path = $"project/importedfile/{Guid.NewGuid()}.dat";
var fileStream = System.IO.File.Open(file.path, FileMode.Open, FileAccess.Read);
var transferProxy = transferProxyFactory.NewProxy(TransferProxyType.Temporary);
transferProxy.Upload(fileStream, s3Path);
var baseUrl = Request.Host.ToUriComponent();
// The QueryString will have values in it, so it's safe to add extra queries with the & as opposed to ?, then &
var callbackUrl = $"http://{baseUrl}/internal/v6/importedfile{Request.QueryString}";
callbackUrl += $"&filename={WebUtility.UrlEncode(file.flowFilename)}&awsFilePath={WebUtility.UrlEncode(s3Path)}";
Logger.LogInformation($"{nameof(BackgroundUpload)}: baseUrl {callbackUrl}");
var executionTimeout = ConfigStore.GetValueInt("PEGASUS_EXECUTION_TIMEOUT_MINS", 5) * 60000;//minutes converted to millisecs
var request = new ScheduleJobRequest
{
Filename = file.flowFilename,
Method = "GET", // match the internal upload Method
Url = callbackUrl,
Timeout = executionTimeout
};
request.SetStringPayload(string.Empty);
var headers = Request.Headers.GetCustomHeaders();
return(await scheduler.ScheduleBackgroundJob(request, headers));
}
public DXFBoundariesRequest(string csibFileData, ImportedFileType fileType, string dxfFileData, DxfUnitsType fileUnits, uint maxBoundaries, bool convertLineStringCoordsToPolygon)
{
CSIBFileData = csibFileData;
FileType = fileType;
DXFFileData = dxfFileData;
FileUnits = fileUnits;
MaxBoundaries = maxBoundaries;
ConvertLineStringCoordsToPolygon = convertLineStringCoordsToPolygon;
}
public async Task <ImportedFileDescriptorSingleResult> InternalImportedFileV6(
[FromQuery] string filename,
[FromQuery] string awsFilePath,
[FromQuery] Guid projectUid,
[FromQuery] ImportedFileType importedFileType,
[FromQuery] DxfUnitsType dxfUnitsType,
[FromQuery] DateTime fileCreatedUtc,
[FromQuery] DateTime fileUpdatedUtc,
[FromQuery] DateTime?surveyedUtc,
[FromServices] ITransferProxyFactory transferProxyFactory,
[FromServices] ISchedulerProxy schedulerProxy)
{
if (importedFileType == ImportedFileType.ReferenceSurface)
{
ServiceExceptionHandler.ThrowServiceException(HttpStatusCode.BadRequest, 122);
}
ImportedFileDescriptorSingleResult importedFileResult = null;
var transferProxy = transferProxyFactory.NewProxy(TransferProxyType.Temporary);
Logger.LogInformation(
$"{nameof(InternalImportedFileV6)}:. filename: {filename} awspath {awsFilePath} projectUid {projectUid} ImportedFileType: {importedFileType} " +
$"DxfUnitsType: {dxfUnitsType} surveyedUtc {(surveyedUtc == null ? "N/A" : surveyedUtc.ToString())}");
// Retrieve the stored file from AWS
var fileResult = await transferProxy.Download(awsFilePath);
if (fileResult == null)
{
ServiceExceptionHandler.ThrowServiceException(HttpStatusCode.InternalServerError, 55);
}
using (var ms = new MemoryStream())
{
// Depending on the size of the file in S3, the stream returned may or may not support seeking
// Which we need to TCC to know the length of the file (can't find the length, if you can't seek).
// To solve this, we have to download the entire stream here and copy to memory.
// Allowing TCC to upload the file.
// Not the best solution for extra large files, but TCC doesn't support uploading without file size AFAIK
fileResult.FileStream.CopyTo(ms);
importedFileResult = await UpsertFileInternal(filename, ms, projectUid, importedFileType, dxfUnitsType,
fileCreatedUtc, fileUpdatedUtc, surveyedUtc, schedulerProxy);
}
Logger.LogInformation(
$"{nameof(InternalImportedFileV6)}: Completed successfully. Response: {JsonConvert.SerializeObject(importedFileResult)}");
return(importedFileResult);
}
public async Task <AddFileResult> GetAddFile(
[FromQuery] Guid projectUid,
[FromQuery] ImportedFileType fileType,
[FromQuery] Guid fileUid,
[FromQuery] string fileDescriptor,
[FromQuery] long fileId,
[FromQuery] DxfUnitsType dxfUnitsType)
{
log.LogDebug($"{nameof(GetAddFile)}: " + Request.QueryString);
var customHeaders = Request.Headers.GetCustomHeaders();
//Do we need to validate fileUid ?
await ClearFilesCaches(projectUid, customHeaders);
dataCache.RemoveByTag(projectUid.ToString());
log.LogInformation($"{nameof(GetAddFile)} returned: " + Response.StatusCode);
return(new AddFileResult(ContractExecutionStatesEnum.ExecutedSuccessfully, "Add file notification successful"));
}
public void Units(DxfUnitsType sourceUnits, DistanceUnitsType destUnits)
{
var reader = new DXFReader(CreateTestPolyline(), sourceUnits);
reader.Should().NotBeNull();
reader.FindEntitiesSection().Should().BeTrue();
var success = reader.GetBoundaryFromPolyLineEntity(true, out var eof, out var boundary);
success.Should().BeTrue();
boundary.Boundary.Points.Should().BeEquivalentTo(new List <FencePoint>
{
new FencePoint(0, 0),
new FencePoint(10 * UnitUtils.DistToMeters(destUnits), 0),
new FencePoint(0, 10 * UnitUtils.DistToMeters(destUnits))
});
}
/// <summary>
/// Create instance of CreateImportedFile
/// </summary>
public CreateImportedFile(Guid projectUid,
string fileName, FileDescriptor fileDescriptor, ImportedFileType importedFileType,
DateTime?surveyedUtc, DxfUnitsType dxfUnitsType, DateTime fileCreatedUtc, DateTime fileUpdatedUtc,
string dataOceanRootFolder, Guid?parentUid, double?offset, Guid importedFileUid, string dataOceanFileName)
{
ProjectUid = projectUid;
FileName = fileName;
FileDescriptor = fileDescriptor;
ImportedFileType = importedFileType;
SurveyedUtc = surveyedUtc;
DxfUnitsType = dxfUnitsType;
FileCreatedUtc = fileCreatedUtc;
FileUpdatedUtc = fileUpdatedUtc;
DataOceanRootFolder = dataOceanRootFolder;
ParentUid = parentUid;
Offset = offset;
ImportedFileUid = importedFileUid;
DataOceanFileName = dataOceanFileName;
}
/// <summary>
///
/// </summary>
public UpdateImportedFile(
Guid projectUid, long shortRaptorProjectId, ImportedFileType importedFileTypeId,
DateTime?surveyedUtc, DxfUnitsType dxfUnitsTypeId,
DateTime fileCreatedUtc, DateTime fileUpdatedUtc,
FileDescriptor fileDescriptor, Guid importedFileUid, long importedFileId,
string dataOceanRootFolder, double?offset, string dataOceanFileName)
{
ProjectUid = projectUid;
ShortRaptorProjectId = shortRaptorProjectId;
ImportedFileType = importedFileTypeId;
SurveyedUtc = surveyedUtc;
DxfUnitsTypeId = dxfUnitsTypeId;
FileCreatedUtc = fileCreatedUtc;
FileUpdatedUtc = fileUpdatedUtc;
FileDescriptor = fileDescriptor;
ImportedFileUid = importedFileUid;
ImportedFileId = importedFileId;
DataOceanRootFolder = dataOceanRootFolder;
Offset = offset;
DataOceanFileName = dataOceanFileName;
}
/// <summary>
/// Common file processing method used by all importedFile endpoints.
/// </summary>
private async Task <ImportedFileDescriptorSingleResult> UpsertFile(
string tmpFilePath,
string filename,
string projectUid,
ImportedFileType importedFileType,
DxfUnitsType dxfUnitsType,
DateTime fileCreatedUtc,
DateTime fileUpdatedUtc,
DateTime?surveyedUtc,
ISchedulerProxy schedulerProxy)
{
if (!System.IO.File.Exists(tmpFilePath))
{
ServiceExceptionHandler.ThrowServiceException(HttpStatusCode.BadRequest, 55);
}
using (var fileStream = new FileStream(tmpFilePath, FileMode.Open))
{
return(await UpsertFileInternal(filename, fileStream, Guid.Parse(projectUid), importedFileType, dxfUnitsType,
fileCreatedUtc, fileUpdatedUtc, surveyedUtc, schedulerProxy));
}
}
public async Task <ImportedFileDescriptorSingleResult> SyncUpload(
[FromServices] ISchedulerProxy schedulerProxy,
FlowFile file,
[FromQuery] Guid projectUid,
[FromQuery] ImportedFileType importedFileType,
[FromQuery] DxfUnitsType dxfUnitsType,
[FromQuery] DateTime fileCreatedUtc,
[FromQuery] DateTime fileUpdatedUtc,
[FromQuery] DateTime?surveyedUtc)
{
if (importedFileType == ImportedFileType.ReferenceSurface)
{
ServiceExceptionHandler.ThrowServiceException(HttpStatusCode.BadRequest, 122);
}
// Validate the file
FlowJsFileImportDataValidator.ValidateUpsertImportedFileRequest(
file, projectUid, importedFileType, dxfUnitsType, fileCreatedUtc, fileUpdatedUtc, UserEmailAddress, surveyedUtc, null, null);
Logger.LogInformation(
$"{nameof(SyncUpload)}: file: {file.flowFilename} path {file.path} projectUid {projectUid} ImportedFileType: {importedFileType} " +
$"DxfUnitsType: {dxfUnitsType} surveyedUtc {(surveyedUtc == null ? "N/A" : surveyedUtc.ToString())}");
await ValidateFileDoesNotExist(projectUid.ToString(), file.flowFilename, importedFileType, surveyedUtc, null, null);
ContractExecutionResult importedFileResult;
using (var fileStream = System.IO.File.Open(file.path, FileMode.Open, FileAccess.Read))
{
importedFileResult = await UpsertFileInternal(file.flowFilename, fileStream, projectUid, importedFileType, dxfUnitsType,
fileCreatedUtc, fileUpdatedUtc, surveyedUtc, schedulerProxy);
}
Logger.LogInformation(
$"{nameof(SyncUpload)}: Completed successfully. Response: {JsonConvert.SerializeObject(importedFileResult)}");
return(importedFileResult as ImportedFileDescriptorSingleResult);
}
public Task <ImportedFileDescriptorSingleResult> UpsertImportedFileV6(
[FromServices] ISchedulerProxy schedulerProxy,
FlowFile file,
[FromQuery] Guid projectUid,
[FromQuery] ImportedFileType importedFileType,
[FromQuery] DxfUnitsType dxfUnitsType,
[FromQuery] DateTime fileCreatedUtc,
[FromQuery] DateTime fileUpdatedUtc,
[FromQuery] DateTime?surveyedUtc = null)
{
if (importedFileType == ImportedFileType.ReferenceSurface)
{
ServiceExceptionHandler.ThrowServiceException(HttpStatusCode.BadRequest, 122);
}
FlowJsFileImportDataValidator.ValidateUpsertImportedFileRequest(file, projectUid, importedFileType, dxfUnitsType, fileCreatedUtc,
fileUpdatedUtc, UserEmailAddress, surveyedUtc, null, null);
Logger.LogInformation(
$"{nameof(UpsertImportedFileV6)}: file: {JsonConvert.SerializeObject(file)} projectUid {projectUid} ImportedFileType: {importedFileType} DxfUnitsType: {dxfUnitsType} surveyedUtc {(surveyedUtc == null ? "N/A" : surveyedUtc.ToString())}");
return(UpsertFile(file.path, file.flowFilename, projectUid.ToString(), importedFileType, dxfUnitsType, fileCreatedUtc, fileUpdatedUtc, surveyedUtc, schedulerProxy));
}
public ContractExecutionResult DummyAddFileGet(
[FromQuery] Guid projectUid,
[FromQuery] ImportedFileType fileType,
[FromQuery] Guid fileUid,
[FromQuery] string fileDescriptor,
[FromQuery] long fileId,
[FromQuery] DxfUnitsType dXfUnitsType)
{
var hasDxfTiles = fileType == ImportedFileType.Linework || fileType == ImportedFileType.Alignment ||
fileType == ImportedFileType.DesignSurface;
var res = new AddFileResult(ContractExecutionStatesEnum.ExecutedSuccessfully, ContractExecutionResult.DefaultMessage)
{
MinZoomLevel = hasDxfTiles ? 15 : 0,
MaxZoomLevel = hasDxfTiles ? 19 : 0,
FileDescriptor = JsonConvert.DeserializeObject <FileDescriptor>(fileDescriptor),
FileUid = fileUid,
UserEmailAddress = "*****@*****.**"
};
var message =
$"DummyAddFileGet: res {res}. projectUid {projectUid} fileType {fileType} fileUid {fileUid} fileDescriptor {fileDescriptor} fileId {fileId} dXfUnitsType {dXfUnitsType}";
Console.WriteLine(message);
return(res);
}
/// <summary>
/// Creates an associated DXF file
/// </summary>
/// <param name="projectId">The id of the project to which the file belongs</param>
/// <param name="fileDescr">The original file for which the associated file is created</param>
/// <param name="suffix">The suffix applied to the file name to get the generated file name</param>
/// <param name="userUnits">The user units preference</param>
private async Task <bool> CreateDxfFile(long projectId, FileDescriptor fileDescr, string suffix, DxfUnitsType userUnits)
{
const double ImperialFeetToMetres = 0.3048;
const double USFeetToMetres = 0.304800609601;
//NOTE: For alignment files only (not surfaces), there are labels generated as part of the DXF file.
//They need to be in the user units.
double interval;
TVLPDDistanceUnits raptorUnits;
switch (userUnits)
{
case DxfUnitsType.ImperialFeet:
raptorUnits = TVLPDDistanceUnits.vduImperialFeet;
interval = 300 * ImperialFeetToMetres;
break;
case DxfUnitsType.Meters:
raptorUnits = TVLPDDistanceUnits.vduMeters;
interval = 100;
break;
case DxfUnitsType.UsSurveyFeet:
default:
raptorUnits = TVLPDDistanceUnits.vduUSSurveyFeet;
interval = 300 * USFeetToMetres;
break;
}
log.LogDebug("Getting DXF design boundary from Raptor");
raptorClient.GetDesignBoundary(
DesignProfiler.ComputeDesignBoundary.RPC.__Global.Construct_CalculateDesignBoundary_Args
(projectId,
fileDescr.DesignDescriptor(configStore, log, 0, 0),
DesignProfiler.ComputeDesignBoundary.RPC.TDesignBoundaryReturnType.dbrtDXF,
interval, raptorUnits, 0), out var memoryStream, out var designProfilerResult);
if (memoryStream != null)
{
return(await PutFile(projectId, fileDescr, suffix, FileUtils.DXF_FILE_EXTENSION, memoryStream, memoryStream.Length));
}
else
{
log.LogWarning("Failed to generate DXF boundary for file {0} for project {1}. Raptor error {2}", fileDescr.FileName, projectId, designProfilerResult);
//We need gracefully fail here as the file may be imported to an empty datamodel
return(false);
/*throw new ServiceException(HttpStatusCode.BadRequest, new ContractExecutionResult(
* ContractExecutionStatesEnum.FailedToGetResults,
* string.Format("Failed to create " + FileUtils.DXF_FILE_EXTENSION + " file with error: {0}",
* ContractExecutionStates.FirstNameWithOffset((int)designProfilerResult))));*/
}
}
public async Task <AddFileResult> AddFile(Guid projectUid, ImportedFileType fileType, Guid fileUid, string fileDescriptor, long fileId, DxfUnitsType dxfUnitsType, IHeaderDictionary customHeaders = null)
{
log.LogDebug($"{nameof(AddFile)} projectUid: {projectUid} fileUid: {fileUid} fileDescriptor: {fileDescriptor} fileId: {fileId} dxfUnitsType: {dxfUnitsType}");
var queryParams = new List <KeyValuePair <string, string> >
{
new KeyValuePair <string, string>("projectUid", projectUid.ToString()), new KeyValuePair <string, string>("fileType", fileType.ToString()),
new KeyValuePair <string, string>("fileUid", fileUid.ToString()), new KeyValuePair <string, string>("fileDescriptor", fileDescriptor),
new KeyValuePair <string, string>("fileId", fileId.ToString()), new KeyValuePair <string, string>("dxfUnitsType", dxfUnitsType.ToString())
};
return(await NotifyFile <AddFileResult>("/notification/addfile", queryParams, customHeaders));
}
/// <summary>
/// Validate the Create request e.g that the file has been uploaded and parameters are as expected.
/// </summary>
public static void ValidateUpsertImportedFileRequest(Guid projectUid, ImportedFileType importedFileType, DxfUnitsType dxfUnitsType, DateTime fileCreatedUtc,
DateTime fileUpdatedUtc, string importedBy, DateTime?surveyedUtc, string filename, Guid?parentUid, double?offset)
{
if (projectUid == Guid.Empty)
{
throw new ServiceException(HttpStatusCode.BadRequest,
new ContractExecutionResult(ProjectErrorCodesProvider.GetErrorNumberwithOffset(5),
ProjectErrorCodesProvider.FirstNameWithOffset(5)));
}
if (!Enum.IsDefined(typeof(ImportedFileType), importedFileType))
{
throw new ServiceException(HttpStatusCode.BadRequest,
new ContractExecutionResult(ProjectErrorCodesProvider.GetErrorNumberwithOffset(30),
ProjectErrorCodesProvider.FirstNameWithOffset(30)));
}
var validType = (importedFileType >= ImportedFileType.Linework && importedFileType <= ImportedFileType.Alignment) ||
importedFileType == ImportedFileType.ReferenceSurface || importedFileType == ImportedFileType.GeoTiff;
if (!validType)
{
throw new ServiceException(HttpStatusCode.BadRequest,
new ContractExecutionResult(ProjectErrorCodesProvider.GetErrorNumberwithOffset(30),
ProjectErrorCodesProvider.FirstNameWithOffset(31)));
}
if (importedFileType != ImportedFileType.ReferenceSurface)
{
var fileExtension = Path.GetExtension(filename).ToLower();
if (!(importedFileType == ImportedFileType.Linework && fileExtension == ".dxf" ||
importedFileType == ImportedFileType.DesignSurface && fileExtension == ".ttm" ||
importedFileType == ImportedFileType.SurveyedSurface && fileExtension == ".ttm" ||
importedFileType == ImportedFileType.Alignment && fileExtension == ".svl" ||
importedFileType == ImportedFileType.GeoTiff && fileExtension == ".tif"))
{
throw new ServiceException(HttpStatusCode.BadRequest,
new ContractExecutionResult(ProjectErrorCodesProvider.GetErrorNumberwithOffset(32),
ProjectErrorCodesProvider.FirstNameWithOffset(32) + $"imported filetype: {importedFileType} and extension is {fileExtension} "));
}
}
if (!Enum.IsDefined(typeof(DxfUnitsType), dxfUnitsType))
{
throw new ServiceException(HttpStatusCode.BadRequest,
new ContractExecutionResult(ProjectErrorCodesProvider.GetErrorNumberwithOffset(75),
ProjectErrorCodesProvider.FirstNameWithOffset(75)));
}
if (importedFileType == ImportedFileType.Linework && (dxfUnitsType < DxfUnitsType.Meters || dxfUnitsType > DxfUnitsType.UsSurveyFeet))
{
throw new ServiceException(HttpStatusCode.BadRequest,
new ContractExecutionResult(ProjectErrorCodesProvider.GetErrorNumberwithOffset(75),
ProjectErrorCodesProvider.FirstNameWithOffset(76)));
}
if (fileCreatedUtc < DateTime.UtcNow.AddYears(-30) || fileCreatedUtc > DateTime.UtcNow.AddDays(2))
{
throw new ServiceException(HttpStatusCode.BadRequest,
new ContractExecutionResult(ProjectErrorCodesProvider.GetErrorNumberwithOffset(33),
ProjectErrorCodesProvider.FirstNameWithOffset(33)));
}
if (fileUpdatedUtc < DateTime.UtcNow.AddYears(-30) || fileUpdatedUtc > DateTime.UtcNow.AddDays(2))
{
throw new ServiceException(HttpStatusCode.BadRequest,
new ContractExecutionResult(ProjectErrorCodesProvider.GetErrorNumberwithOffset(34),
ProjectErrorCodesProvider.FirstNameWithOffset(34)));
}
if (string.IsNullOrEmpty(importedBy))
{
throw new ServiceException(HttpStatusCode.BadRequest,
new ContractExecutionResult(ProjectErrorCodesProvider.GetErrorNumberwithOffset(35),
ProjectErrorCodesProvider.FirstNameWithOffset(35)));
}
if ((importedFileType == ImportedFileType.SurveyedSurface || importedFileType == ImportedFileType.GeoTiff) && surveyedUtc == null)
{
throw new ServiceException(HttpStatusCode.BadRequest,
new ContractExecutionResult(ProjectErrorCodesProvider.GetErrorNumberwithOffset(36),
ProjectErrorCodesProvider.FirstNameWithOffset(36)));
}
if (importedFileType == ImportedFileType.ReferenceSurface && (parentUid == null || offset == null || offset.Value == 0))
{
throw new ServiceException(HttpStatusCode.BadRequest,
new ContractExecutionResult(ProjectErrorCodesProvider.GetErrorNumberwithOffset(118),
ProjectErrorCodesProvider.FirstNameWithOffset(118)));
}
}
public async void Binary_DXF_Boundaries_UnderLimit_Closed(string fileName, DxfUnitsType units, int expectedBoundaryCount, int firstBoundaryVertexCount, string expectedName, DXFLineWorkBoundaryType expectedType)
{
TestAFile(fileName, units, expectedBoundaryCount, firstBoundaryVertexCount, expectedName, expectedType, false);
}
/// <summary>
/// Creates a job request for generating raster tiles
/// </summary>
public static JobRequest CreateRequest(ImportedFileType importedFileType, string customerUid, string projectUid,
string importedFileUid, string dataOceanRootFolder, string fileName, string dcFileName = null, DxfUnitsType
dxfUnitsType = DxfUnitsType.Meters, DateTime?surveyedUtc = null)
{
TileGenerationRequest runParams;
Guid jobUid;
switch (importedFileType)
{
case ImportedFileType.GeoTiff:
runParams = new TileGenerationRequest();
jobUid = GeoTiffTileGenerationJob.VSSJOB_UID;
break;
case ImportedFileType.Linework:
runParams = new DxfTileGenerationRequest
{
DcFileName = dcFileName,
DxfUnitsType = dxfUnitsType
};
jobUid = DxfTileGenerationJob.VSSJOB_UID;
break;
default:
throw new NotImplementedException();
}
runParams.CustomerUid = Guid.Parse(customerUid);
runParams.ProjectUid = Guid.Parse(projectUid);
runParams.ImportedFileUid = Guid.Parse(importedFileUid);
runParams.DataOceanRootFolder = dataOceanRootFolder;
runParams.FileName = fileName;
return(new JobRequest
{
JobUid = jobUid,
RunParameters = runParams
});
}
/// <summary>
/// Generates DXF tiles using the Pegasus API and stores them in the data ocean.
/// </summary>
/// <param name="dcFileName">The path and file name of the coordinate system file</param>
/// <param name="dxfFileName">The path and file name of the DXF file</param>
/// <param name="dxfUnitsType">The units of the DXF file</param>
/// <param name="customHeaders"></param>
/// <param name="setJobIdAction"></param>
/// <returns>Metadata for the generated tiles including the zoom range</returns>
public async Task <TileMetadata> GenerateDxfTiles(string dcFileName, string dxfFileName, DxfUnitsType dxfUnitsType, IHeaderDictionary customHeaders, Action <IHeaderDictionary> setJobIdAction)
{
Log.LogInformation($"{nameof(GenerateDxfTiles)}: dcFileName={dcFileName}, dxfFileName={dxfFileName}, dxfUnitsType={dxfUnitsType}");
//Get the DataOcean file ids.
var dcFileId = await dataOceanClient.GetFileId(dcFileName, customHeaders);
if (dcFileId == null)
{
var message = $"Failed to find coordinate system file {dcFileName}. Has it been uploaded successfully?";
throw new ServiceException(HttpStatusCode.InternalServerError,
new ContractExecutionResult(ContractExecutionStatesEnum.InternalProcessingError, message));
}
var dxfFileId = await dataOceanClient.GetFileId(dxfFileName, customHeaders);
if (dxfFileId == null)
{
var message = $"Failed to find DXF file {dxfFileName}. Has it been uploaded successfully?";
throw new ServiceException(HttpStatusCode.InternalServerError,
new ContractExecutionResult(ContractExecutionStatesEnum.InternalProcessingError, message));
}
//Get the Pegasus units
var pegasusUnits = PegasusUnitsType.Metre;
switch (dxfUnitsType)
{
case DxfUnitsType.Meters:
break;
case DxfUnitsType.UsSurveyFeet:
pegasusUnits = PegasusUnitsType.USSurveyFoot;
break;
case DxfUnitsType.ImperialFeet:
pegasusUnits = PegasusUnitsType.BritishFoot;
break;
}
//1. Create an execution
var createExecutionMessage = new CreateExecutionMessage
{
Execution = new PegasusExecution
{
ProcedureId = dxfProcedureId,
Parameters = new DxfPegasusExecutionParameters
{
DcFileId = dcFileId.Value,
DxfFileId = dxfFileId.Value,
MaxZoom = maxZoomLevel.ToString(),
TileType = TILE_TYPE,
AngularUnit = AngularUnitsType.Degree.ToString(),
PlaneUnit = pegasusUnits.ToString(),
VerticalUnit = pegasusUnits.ToString()
}
}
};
return(await GenerateTiles(dxfFileName, createExecutionMessage, customHeaders, setJobIdAction));
}
/// <summary>
/// Common file processing method used by all importedFile endpoints.
/// </summary>
protected async Task <ImportedFileDescriptorSingleResult> UpsertFileInternal(
string filename,
Stream fileStream,
Guid projectUid,
ImportedFileType importedFileType,
DxfUnitsType dxfUnitsType,
DateTime fileCreatedUtc,
DateTime fileUpdatedUtc,
DateTime?surveyedUtc,
ISchedulerProxy schedulerProxy,
Guid?parentUid = null,
double?offset = null)
{
ImportedFileDescriptorSingleResult importedFile = null;
var existing = await ImportedFileRequestDatabaseHelper
.GetImportedFileForProject
(projectUid.ToString(), filename, importedFileType, surveyedUtc,
Logger, ProjectRepo, offset, parentUid)
.ConfigureAwait(false);
var creating = existing == null;
Logger.LogInformation(
creating
? $"{nameof(UpsertFileInternal)}. file doesn't exist already in DB: {filename} projectUid {projectUid} ImportedFileType: {importedFileType} surveyedUtc {(surveyedUtc == null ? "N/A" : surveyedUtc.ToString())} parentUid {parentUid} offset: {offset}"
: $"{nameof(UpsertFileInternal)}. file exists already in DB. Will be updated: {JsonConvert.SerializeObject(existing)}");
FileDescriptor fileDescriptor = null;
var importedFileUid = creating ? Guid.NewGuid() : Guid.Parse(existing.ImportedFileUid);
var dataOceanFileName = DataOceanFileUtil.DataOceanFileName(filename,
importedFileType == ImportedFileType.SurveyedSurface || importedFileType == ImportedFileType.GeoTiff,
importedFileUid, surveyedUtc);
if (importedFileType == ImportedFileType.ReferenceSurface)
{
//FileDescriptor not used for reference surface but validation requires values
fileDescriptor = FileDescriptor.CreateFileDescriptor("Not applicable", "Not applicable", filename);
}
else
{
if (IsTRexDesignFileType(importedFileType))
{
fileDescriptor = ProjectRequestHelper.WriteFileToS3Repository(
fileStream, projectUid.ToString(), filename,
importedFileType == ImportedFileType.SurveyedSurface, surveyedUtc,
Logger, ServiceExceptionHandler, persistantTransferProxyFactory.NewProxy(TransferProxyType.DesignImport));
}
//This is needed for ATs.
fileDescriptor = FileDescriptor.CreateFileDescriptor(
FileSpaceId,
$"/{CustomerUid}/{projectUid}",
filename);
if (importedFileType == ImportedFileType.Linework || importedFileType == ImportedFileType.GeoTiff)
{
//save copy to DataOcean
await DataOceanHelper.WriteFileToDataOcean(
fileStream, DataOceanRootFolderId, CustomerUid, projectUid.ToString(), dataOceanFileName,
Logger, ServiceExceptionHandler, DataOceanClient, Authorization, importedFileUid, ConfigStore);
}
}
if (creating)
{
var createImportedFile = new CreateImportedFile(
projectUid, filename, fileDescriptor, importedFileType, surveyedUtc, dxfUnitsType,
fileCreatedUtc, fileUpdatedUtc, DataOceanRootFolderId, parentUid, offset, importedFileUid, dataOceanFileName);
importedFile = await WithServiceExceptionTryExecuteAsync(() =>
RequestExecutorContainerFactory
.Build <CreateImportedFileExecutor>(
LoggerFactory, ConfigStore, ServiceExceptionHandler, CustomerUid, UserId, UserEmailAddress, customHeaders,
productivity3dV2ProxyCompaction : Productivity3dV2ProxyCompaction,
persistantTransferProxyFactory : persistantTransferProxyFactory, tRexImportFileProxy : tRexImportFileProxy,
projectRepo : ProjectRepo, dataOceanClient : DataOceanClient, authn : Authorization, schedulerProxy : schedulerProxy,
cwsProjectClient : CwsProjectClient)
.ProcessAsync(createImportedFile)
) as ImportedFileDescriptorSingleResult;
Logger.LogInformation(
$"{nameof(UpsertFileInternal)}: Create completed successfully. Response: {JsonConvert.SerializeObject(importedFile)}");
}
else
{
// this also validates that this customer has access to the projectUid
var project = await ProjectRequestHelper.GetProject(projectUid, new Guid(CustomerUid), new Guid(UserId), Logger, ServiceExceptionHandler, CwsProjectClient, customHeaders);
var importedFileUpsertEvent = new UpdateImportedFile(
projectUid, project.ShortRaptorProjectId, importedFileType,
(importedFileType == ImportedFileType.SurveyedSurface || importedFileType == ImportedFileType.GeoTiff)
? surveyedUtc
: null,
dxfUnitsType, fileCreatedUtc, fileUpdatedUtc, fileDescriptor,
Guid.Parse(existing?.ImportedFileUid), existing.ImportedFileId,
DataOceanRootFolderId, offset, dataOceanFileName);
importedFile = await WithServiceExceptionTryExecuteAsync(() =>
RequestExecutorContainerFactory
.Build <UpdateImportedFileExecutor>(
LoggerFactory, ConfigStore, ServiceExceptionHandler, CustomerUid, UserId, UserEmailAddress, customHeaders,
productivity3dV2ProxyCompaction : Productivity3dV2ProxyCompaction,
tRexImportFileProxy : tRexImportFileProxy,
projectRepo : ProjectRepo, dataOceanClient : DataOceanClient, authn : Authorization, schedulerProxy : schedulerProxy,
cwsProjectClient : CwsProjectClient)
.ProcessAsync(importedFileUpsertEvent)
) as ImportedFileDescriptorSingleResult;
Logger.LogInformation(
$"{nameof(UpsertFileInternal)}: Update completed successfully. Response: {JsonConvert.SerializeObject(importedFile)}");
}
await NotificationHubClient.Notify(new ProjectChangedNotification(projectUid));
return(importedFile);
}
public static DXFUtilitiesResult RequestBoundariesFromLineWork(MemoryStream dxfFileStream, DxfUnitsType units, uint maxBoundariesToProcess, bool convertLineStringCoordsToPolygon, out PolyLineBoundaries boundaries)
{
boundaries = null;
var reader = new DXFReader(dxfFileStream, units);
try
{
if (!reader.FindEntitiesSection())
{
return(DXFUtilitiesResult.NoEntitiesSectionFound);
}
var extractor = new PolyLineBoundaryExtractor(reader);
var result = extractor.Extract(units, maxBoundariesToProcess, convertLineStringCoordsToPolygon);
if (result == DXFUtilitiesResult.Ok)
{
boundaries = extractor.Boundaries;
}
return(result);
}
catch (FormatException e)
{
Log.LogError(e, "DXF file not in expected format");
return(DXFUtilitiesResult.UnknownFileFormat);
}
catch (Exception e)
{
Log.LogError(e, "Exception processing DXF file");
throw;
}
}
public PolyLineBoundaries(DxfUnitsType units, uint maxBoundariesToProcess)
{
Units = units;
Boundaries = new List <PolyLineBoundary>((int)maxBoundariesToProcess);
}
public static DXFUtilitiesResult RequestBoundariesFromLineWork(string dxfFileContext, DxfUnitsType units, uint maxBoundariesToProcess, bool convertLineStringCoordsToPolygon, out PolyLineBoundaries boundaries)
{
using var stream = new MemoryStream(Convert.FromBase64String(dxfFileContext));
return(DXFFileUtilities.RequestBoundariesFromLineWork(stream, units, maxBoundariesToProcess, convertLineStringCoordsToPolygon, out boundaries));
}
public static AlignmentLineworkRequest Create(Guid projectUid, long projectId, MasterDataModels.FileDescriptor fileDescr, DxfUnitsType userUnits)
{
return(new AlignmentLineworkRequest
{
ProjectUid = projectUid,
ProjectId = projectId,
FileDescriptor = fileDescr,
UserUnits = userUnits
});
}
/// <summary>
/// Validate the Create request e.g that the file has been uploaded and parameters are as expected.
/// </summary>
public static void ValidateUpsertImportedFileRequest(FlowFile file, Guid projectUid,
ImportedFileType importedFileType, DxfUnitsType dxfUnitsType,
DateTime fileCreatedUtc, DateTime fileUpdatedUtc,
string importedBy, DateTime?surveyedUtc, Guid?parentUid, double?offset)
{
if (file == null)
{
throw new ServiceException(HttpStatusCode.BadRequest,
new ContractExecutionResult(ProjectErrorCodesProvider.GetErrorNumberwithOffset(27),
ProjectErrorCodesProvider.FirstNameWithOffset(27)));
}
if (file.flowFilename.Length > MAX_FILE_NAME_LENGTH || string.IsNullOrEmpty(file.flowFilename) ||
file.flowFilename.IndexOfAny(Path.GetInvalidPathChars()) > 0)
{
throw new ServiceException(HttpStatusCode.BadRequest,
new ContractExecutionResult(ProjectErrorCodesProvider.GetErrorNumberwithOffset(28),
ProjectErrorCodesProvider.FirstNameWithOffset(28)));
}
if (string.IsNullOrEmpty(file.path) || file.path.IndexOfAny(Path.GetInvalidPathChars()) > 0)
{
throw new ServiceException(HttpStatusCode.BadRequest,
new ContractExecutionResult(ProjectErrorCodesProvider.GetErrorNumberwithOffset(29),
ProjectErrorCodesProvider.FirstNameWithOffset(29)));
}
ValidateUpsertImportedFileRequest(projectUid, importedFileType, dxfUnitsType, fileCreatedUtc, fileUpdatedUtc, importedBy, surveyedUtc, file.flowFilename, parentUid, offset);
}
