/// <summary>
/// Primary method called to begin analytics computation
/// </summary>
public bool ComputeAnalytics(BaseAnalyticsResponse response)
{
using var processor = DIContext.Obtain <IPipelineProcessorFactory>().NewInstanceNoBuild <SubGridsRequestArgument>(
RequestDescriptor,
SiteModel.ID,
RequestedGridDataType,
response,
Filters,
CutFillDesign,
DIContext.Obtain <Func <PipelineProcessorTaskStyle, ITRexTask> >()(PipelineProcessorTaskStyle.AggregatedPipelined),
DIContext.Obtain <Func <PipelineProcessorPipelineStyle, ISubGridPipelineBase> >()(PipelineProcessorPipelineStyle.DefaultAggregative),
DIContext.Obtain <IRequestAnalyser>(),
IncludeSurveyedSurfaces,
CutFillDesign?.DesignID != Guid.Empty,
BoundingIntegerExtent2D.Inverted(),
LiftParams
);
// Assign the provided aggregator into the pipelined sub grid task
((IAggregatedPipelinedSubGridTask)processor.Task).Aggregator = Aggregator;
if (!processor.Build())
{
Log.LogError($"Failed to build pipeline processor for request to model {SiteModel.ID}");
return(false);
}
processor.Process();
return(response.ResultStatus == RequestErrorStatus.OK);
}
private ProgressiveVolumesSubGridRetriever MakeANewRetriever(ISiteModel siteModel)
{
var retriever = new ProgressiveVolumesSubGridRetriever
(siteModel,
GridDataType.ProgressiveVolumes,
siteModel.PrimaryStorageProxy,
new CombinedFilter(),
new CellPassAttributeFilterProcessingAnnex(),
false,
BoundingIntegerExtent2D.Inverted(),
true,
int.MaxValue,
new AreaControlSet(),
new FilteredValuePopulationControl(),
new SubGridTreeBitMask(),
new OverrideParameters(),
new LiftParameters()
);
retriever.StartDate = new DateTime(2020, 1, 1, 0, 0, 0);
retriever.EndDate = new DateTime(2020, 1, 1, 1, 0, 0);
retriever.Interval = new TimeSpan(0, 10, 0);
return(retriever);
}
public void NewInstance()
{
var factory = new PipelineProcessorFactory();
var processor = factory.NewInstance <SubGridsRequestArgument>(
Guid.NewGuid(),
Guid.NewGuid(),
GridDataType.Height,
new PatchRequestResponse(),
new FilterSet(new CombinedFilter()),
new DesignOffset(),
DIContext.Obtain <Func <PipelineProcessorTaskStyle, ITRexTask> >()(PipelineProcessorTaskStyle.PatchExport),
DIContext.Obtain <Func <PipelineProcessorPipelineStyle, ISubGridPipelineBase> >()(PipelineProcessorPipelineStyle.DefaultProgressive),
DIContext.Obtain <IRequestAnalyser>(),
false,
false,
BoundingIntegerExtent2D.Inverted(),
new LiftParameters());
processor.Should().BeNull("because there is no site model");
// Configure the request analyser to return a single page of results.
// processor.RequestAnalyser.SinglePageRequestNumber = 1;
// processor.RequestAnalyser.SinglePageRequestSize = 10;
// processor.RequestAnalyser.SubmitSinglePageOfRequests = true;
// processor.Build().Should().BeFalse("because there is no sitemodel");
}
/// <summary>
/// Executor that implements requesting and rendering sub grid information to create the rendered tile
/// </summary>
/// <returns></returns>
public async Task <bool> ExecuteAsync()
{
Log.LogInformation($"Performing Execute for DataModel:{DataModelID}, Mode={Mode}, RequestingNodeID={RequestingTRexNodeID}");
ApplicationServiceRequestStatistics.Instance.NumSubgridPageRequests.Increment();
Guid RequestDescriptor = Guid.NewGuid();
using (var processor = DIContext.Obtain <IPipelineProcessorFactory>().NewInstanceNoBuild <SubGridsRequestArgument>(
RequestDescriptor,
DataModelID,
// Patch requests always want time with height information
Mode == DisplayMode.Height ? GridDataType.HeightAndTime : GridDataFromModeConverter.Convert(Mode),
PatchSubGridsResponse,
Filters,
CutFillDesign,
DIContext.Obtain <Func <PipelineProcessorTaskStyle, ITRexTask> >()(PipelineProcessorTaskStyle.PatchExport),
DIContext.Obtain <Func <PipelineProcessorPipelineStyle, ISubGridPipelineBase> >()(PipelineProcessorPipelineStyle.DefaultProgressive),
DIContext.Obtain <IRequestAnalyser>(),
Rendering.Utilities.DisplayModeRequireSurveyedSurfaceInformation(Mode) &&
Rendering.Utilities.FilterRequireSurveyedSurfaceInformation(Filters),
Rendering.Utilities.RequestRequiresAccessToDesignFileExistenceMap(Mode, CutFillDesign),
BoundingIntegerExtent2D.Inverted(),
LiftParams))
{
// Set the surface TRexTask parameters for progressive processing
processor.Task.TRexNodeID = RequestingTRexNodeID;
// Configure the request analyser to return a single page of results.
processor.RequestAnalyser.SinglePageRequestNumber = DataPatchPageNumber;
processor.RequestAnalyser.SinglePageRequestSize = DataPatchPageSize;
processor.RequestAnalyser.SubmitSinglePageOfRequests = true;
if (!processor.Build())
{
Log.LogError($"Failed to build pipeline processor for request to model {DataModelID}");
return(false);
}
// If this is the first page requested then count the total number of patches required for all sub grids to be returned
if (DataPatchPageNumber == 0)
{
PatchSubGridsResponse.TotalNumberOfPagesToCoverFilteredData =
(int)Math.Truncate(Math.Ceiling(processor.RequestAnalyser.CountOfSubGridsThatWillBeSubmitted() / (double)DataPatchPageSize));
}
processor.Process();
if (PatchSubGridsResponse.ResultStatus == RequestErrorStatus.OK)
{
PatchSubGridsResponse.SubGrids = ((PatchTask)processor.Task).PatchSubGrids;
}
return(true);
}
}
public void SubGridForCaching_IgnoresFilterMask_WithPartialNonOverlappingOverrideMaskRetriction()
{
var siteModel = BuildModelForSubGridRequest();
var retriever = new SubGridRetriever(siteModel,
GridDataType.Height,
siteModel.PrimaryStorageProxy,
new CombinedFilter(),
new CellPassAttributeFilterProcessingAnnex(),
true, // Has override mask
BoundingIntegerExtent2D.Inverted(),
true, // prepareGridForCacheStorageIfNoSieving
1000,
AreaControlSet.CreateAreaControlSet(),
new FilteredValuePopulationControl(),
siteModel.ExistenceMap,
new OverrideParameters(),
new LiftParameters()
);
var clientGrid = ClientLeafSubGridFactoryFactory.CreateClientSubGridFactory().GetSubGridEx
(GridDataType.Height, SubGridTreeConsts.DefaultCellSize, SubGridTreeConsts.SubGridTreeLevels,
SubGridTreeConsts.DefaultIndexOriginOffset, SubGridTreeConsts.DefaultIndexOriginOffset);
var overrideMask = new SubGridTreeBitmapSubGridBits(SubGridBitsCreationOptions.Unfilled);
overrideMask[10, 10] = true; // This does not overlap the filter but should still return a result
var result = retriever.RetrieveSubGrid(clientGrid, overrideMask, out var seiveFilterInUse,
() =>
{
clientGrid.FilterMap.Clear();
clientGrid.FilterMap[0, 0] = true;
clientGrid.ProdDataMap.Fill();
return(ServerRequestResult.NoError);
});
result.Should().Be(ServerRequestResult.NoError);
seiveFilterInUse.Should().BeFalse();
clientGrid.FilterMap.CountBits().Should().Be(1); // Only asking for the one cell...
clientGrid.CountNonNullCells().Should().Be(1024); // All cells should be returned
}
public void NewInstance_NoBuild()
{
var factory = new PipelineProcessorFactory();
var processor = factory.NewInstanceNoBuild <SubGridsRequestArgument>(
Guid.NewGuid(),
Guid.NewGuid(),
GridDataType.Height,
new PatchRequestResponse(),
new FilterSet(new CombinedFilter()),
new DesignOffset(),
DIContext.Obtain <Func <PipelineProcessorTaskStyle, ITRexTask> >()(PipelineProcessorTaskStyle.PatchExport),
DIContext.Obtain <Func <PipelineProcessorPipelineStyle, ISubGridPipelineBase> >()(PipelineProcessorPipelineStyle.DefaultProgressive),
DIContext.Obtain <IRequestAnalyser>(),
false,
false,
BoundingIntegerExtent2D.Inverted(),
new LiftParameters());
processor.Should().NotBeNull();
}
/// <summary>
/// Executor that implements creation of the TIN surface
/// </summary>
public async Task <bool> ExecuteAsync()
{
Log.LogInformation($"Performing Execute for DataModel:{_dataModelId}");
try
{
var requestDescriptor = Guid.NewGuid();
var siteModel = DIContext.Obtain <ISiteModels>().GetSiteModel(_dataModelId);
if (siteModel == null)
{
Log.LogError($"Failed to obtain site model for {_dataModelId}");
return(false);
}
var datastore = new GenericSubGridTree <float, GenericLeafSubGrid <float> >(siteModel.Grid.NumLevels, siteModel.CellSize);
// Provide the processor with a customised request analyser configured to return a set of sub grids. These sub grids
// are the feed stock for the generated TIN surface
using (var processor = DIContext.Obtain <IPipelineProcessorFactory>().NewInstanceNoBuild <SubGridsRequestArgument>(
requestDescriptor,
_dataModelId,
GridDataFromModeConverter.Convert(DisplayMode.Height),
SurfaceSubGridsResponse,
_filters,
new DesignOffset(),
DIContext.Obtain <Func <PipelineProcessorTaskStyle, ITRexTask> >()(PipelineProcessorTaskStyle.SurfaceExport),
DIContext.Obtain <Func <PipelineProcessorPipelineStyle, ISubGridPipelineBase> >()(PipelineProcessorPipelineStyle.DefaultProgressive),
DIContext.Obtain <IRequestAnalyser>(),
Rendering.Utilities.DisplayModeRequireSurveyedSurfaceInformation(DisplayMode.Height) && Rendering.Utilities.FilterRequireSurveyedSurfaceInformation(_filters),
false, //Rendering.Utilities.RequestRequiresAccessToDesignFileExistenceMap(DisplayMode.Height),
BoundingIntegerExtent2D.Inverted(),
_liftParams))
{
// Set the surface TRexTask parameters for progressive processing
processor.Task.TRexNodeID = RequestingTRexNodeID;
if (!processor.Build())
{
Log.LogError($"Failed to build pipeline processor for request to model {_dataModelId}");
return(false);
}
processor.Process();
if (SurfaceSubGridsResponse.ResultStatus != RequestErrorStatus.OK)
{
Log.LogError($"Sub grids response status not OK: {SurfaceSubGridsResponse.ResultStatus}");
return(false);
}
// Create the TIN decimator and populate it with the retrieved sub grids
foreach (var subGrid in ((SurfaceTask)processor.Task).SurfaceSubgrids)
{
if (!(datastore.ConstructPathToCell(subGrid.OriginX, subGrid.OriginY, SubGridPathConstructionType.CreatePathToLeaf) is INodeSubGrid newGridNode))
{
Log.LogError($"Result from data store ConstructPathToCell({subGrid.OriginX}, {subGrid.OriginY}) was null. Aborting...");
return(false);
}
subGrid.Owner = datastore;
newGridNode.GetSubGridCellIndex(subGrid.OriginX, subGrid.OriginY, out var subGridIndexX, out var subGridIndexY);
newGridNode.SetSubGrid(subGridIndexX, subGridIndexY, subGrid);
}
}
// Obtain the surface export data smoother and apply it to the tree of queried data before passing it to the decimation engine
var dataSmoother = DIContext.Obtain <Func <DataSmootherOperation, IDataSmoother> >()(DataSmootherOperation.SurfaceExport) as ITreeDataSmoother <float>;
datastore = dataSmoother?.Smooth(datastore) ?? datastore;
var extents = DataStoreExtents(datastore);
// Make sure we don't export too large an area due to data way outside project extents
if (extents.Area > Common.Consts.MaxExportAreaM2)
{
// First try and use project boundary extents as our data boundary
var canExport = siteModel.SiteModelExtent.Area > 0 && siteModel.SiteModelExtent.Area < Common.Consts.MaxExportAreaM2;
if (canExport)
{
// still use min max height extents
Log.LogInformation($"Invalid Plan Extent. Data area too large {extents.Area}. Switching to project extents");
extents.MinX = siteModel.SiteModelExtent.MinX;
extents.MinY = siteModel.SiteModelExtent.MinY;
extents.MaxX = siteModel.SiteModelExtent.MaxX;
extents.MaxY = siteModel.SiteModelExtent.MaxY;
}
else
{
Log.LogError($"Invalid Plan Extent. Data area too large {extents.Area}.");
return(false);
}
}
// Decimate the elevations into a grid
var decimator = new GridToTINDecimator(datastore)
{
Tolerance = _tolerance
};
decimator.SetDecimationExtents(extents);
if (!decimator.BuildMesh())
{
Log.LogError($"Decimator returned false with error code: {decimator.BuildMeshFaultCode}");
return(false);
}
// A decimated TIN has been successfully constructed... Return it!
SurfaceSubGridsResponse.TIN = decimator.GetTIN();
}
catch (Exception e)
{
Log.LogError(e, "ExecutePipeline raised Exception:");
return(false);
}
return(true);
}
public bool ExecutePipeline()
{
try
{
if (ActiveDesign != null && (VolumeType == VolumeComputationType.BetweenFilterAndDesign || VolumeType == VolumeComputationType.BetweenDesignAndFilter))
{
if (ActiveDesign == null || ActiveDesign.Design.DesignDescriptor.IsNull)
{
_log.LogError($"No design provided to prod data/design volumes calc for datamodel {SiteModel.ID}");
VolumesRequestResponse.ResultStatus = RequestErrorStatus.NoDesignProvided;
return(false);
}
}
// Note: The execution context is on a compute cluster node already. However, the processor still performs the same function
// within the local context.
using var processor = DIContext.ObtainRequired <IPipelineProcessorFactory>().NewInstanceNoBuild <ProgressiveVolumesSubGridsRequestArgument>
(requestDescriptor: RequestDescriptor,
dataModelID: SiteModel.ID,
gridDataType: GridDataType.ProgressiveVolumes,
response: VolumesRequestResponse,
cutFillDesign: null,
filters: new FilterSet(Filter),
task: DIContext.ObtainRequired <Func <PipelineProcessorTaskStyle, ITRexTask> >()(PipelineProcessorTaskStyle.ProgressiveVolumes),
pipeline: DIContext.ObtainRequired <Func <PipelineProcessorPipelineStyle, ISubGridPipelineBase> >()(PipelineProcessorPipelineStyle.ProgressiveVolumes),
requestAnalyser: DIContext.ObtainRequired <IRequestAnalyser>(),
requestRequiresAccessToDesignFileExistenceMap: false, // Note: RefDesign != null || RefOriginal != null,
requireSurveyedSurfaceInformation: false, //UseSurveyedSurfaces, //_filteredSurveyedSurfaces.Count > 0,
overrideSpatialCellRestriction: BoundingIntegerExtent2D.Inverted(),
liftParams: null
);
// Assign the aggregator into the volumes computation task
if (!(processor.Task is VolumesComputationTask volumesComputationTask))
{
throw new ArgumentException($"Processor task for progressive volumes is not a {nameof(VolumesComputationTask)}, it is '{processor.Task}'");
}
volumesComputationTask.Aggregator = Aggregator;
// Create the initialization lambda to be applied to the sub grid request argument creation in the sub grid
// pipeline processor execution context.
if (processor is IPipelineProcessor <ProgressiveVolumesSubGridsRequestArgument> customProcessor)
{
customProcessor.CustomArgumentInitializer = arg =>
{
arg.StartDate = StartDate;
arg.EndDate = EndDate;
arg.Interval = Interval;
};
}
processor.Task.RequestDescriptor = RequestDescriptor;
if (!processor.Build())
{
_log.LogError($"Failed to build pipeline processor for request to model {SiteModel.ID} with status {processor.Response.ResultStatus}");
VolumesRequestResponse.ResultStatus = processor.Response.ResultStatus;
return(false);
}
processor.Process();
if (processor.Response.ResultStatus != RequestErrorStatus.OK)
{
_log.LogError($"Failed to compute progressive volumes data, for project: {SiteModel.ID}. response: {processor.Response.ResultStatus}.");
VolumesRequestResponse.ResultStatus = processor.Response.ResultStatus;
return(false);
}
return(true);
}
catch (Exception e)
{
_log.LogError(e, "Pipeline processor raised exception");
}
return(false);
}
/// <summary>
/// Executor that implements requesting and rendering grid information to create the grid rows
/// </summary>
/// <returns></returns>
public async Task <bool> ExecuteAsync()
{
Log.LogInformation($"Performing Execute for DataModel:{_griddedReportRequestArgument.ProjectID}");
ApplicationServiceRequestStatistics.Instance.NumSubgridPageRequests.Increment();
var requestDescriptor = Guid.NewGuid();
var task = DIContext.Obtain <Func <PipelineProcessorTaskStyle, ITRexTask> >()(PipelineProcessorTaskStyle.GriddedReport) as GriddedReportTask;
using (var processor = DIContext.Obtain <IPipelineProcessorFactory>().NewInstanceNoBuild <SubGridsRequestArgument>(
requestDescriptor,
_griddedReportRequestArgument.ProjectID,
GridDataType.CellProfile,
GriddedReportRequestResponse,
_griddedReportRequestArgument.Filters,
_griddedReportRequestArgument.ReferenceDesign,
task,
DIContext.Obtain <Func <PipelineProcessorPipelineStyle, ISubGridPipelineBase> >()(PipelineProcessorPipelineStyle.DefaultProgressive),
DIContext.Obtain <IRequestAnalyser>(),
Rendering.Utilities.FilterRequireSurveyedSurfaceInformation(_griddedReportRequestArgument.Filters),
_griddedReportRequestArgument.ReferenceDesign != null && _griddedReportRequestArgument.ReferenceDesign.DesignID != Guid.Empty,
BoundingIntegerExtent2D.Inverted(),
_griddedReportRequestArgument.LiftParams
))
{
// Set the grid TRexTask parameters for progressive processing
processor.Task.RequestDescriptor = requestDescriptor;
processor.Task.TRexNodeID = _griddedReportRequestArgument.TRexNodeID;
processor.Task.GridDataType = GridDataType.CellProfile;
task.ProcessorDelegate = async subGrid => GriddedReportRequestResponse.GriddedReportDataRowList
.AddRange(ExtractRequiredValues(_griddedReportRequestArgument, subGrid));
// report options 0=direction,1=endpoint,2=automatic
if (_griddedReportRequestArgument.GridReportOption == GridReportOption.EndPoint)
{
// Compute the bearing between the two points as a survey (north azimuth, clockwise increasing)
_griddedReportRequestArgument.Azimuth = Math.Atan2(_griddedReportRequestArgument.EndNorthing - _griddedReportRequestArgument.StartNorthing, _griddedReportRequestArgument.EndEasting - _griddedReportRequestArgument.StartEasting);
}
else
{
if (_griddedReportRequestArgument.GridReportOption == GridReportOption.Automatic)
{
// automatic
_griddedReportRequestArgument.Azimuth = 0;
_griddedReportRequestArgument.StartNorthing = 0;
_griddedReportRequestArgument.StartEasting = 0;
}
}
// Avoid starting on cell boundary by applying tiny offset
_griddedReportRequestArgument.StartNorthing = StartGridOffset;
_griddedReportRequestArgument.StartEasting = StartGridOffset;
// Interval will be >= 0.1m and <= 100.0m
processor.Pipeline.AreaControlSet =
new AreaControlSet(false, _griddedReportRequestArgument.GridInterval, _griddedReportRequestArgument.GridInterval,
_griddedReportRequestArgument.StartEasting, _griddedReportRequestArgument.StartNorthing,
_griddedReportRequestArgument.Azimuth);
if (!processor.Build())
{
Log.LogError($"Failed to build pipeline processor for request to model {_griddedReportRequestArgument.ProjectID}");
return(false);
}
processor.Process();
if (GriddedReportRequestResponse.ResultStatus != RequestErrorStatus.OK)
{
throw new ArgumentException($"Unable to obtain data for Gridded report. GriddedReportRequestResponse: {GriddedReportRequestResponse.ResultStatus.ToString()}.");
}
}
return(true);
}
