/// <summary>
/// Builds the pipeline configured per the supplied state ready to execute the request
/// </summary>
public bool Build()
{
// Todo: This method is left as async as a reminder that the GetExistenveMap workflows could either be async (as they
// potentially read from the persistent store), and/or they couild be cached in the site model designs/surveyed surfaces contexts
// See Jira CCSSSCON-1481
try
{
var stopWatch = Stopwatch.StartNew();
// Ensure the task is initialised with the request descriptor
Task.RequestDescriptor = RequestDescriptor;
// Ensure the Task grid data type matches the pipeline processor
Task.GridDataType = GridDataType;
// Introduce the task and the pipeline to each other
Pipeline.PipelineTask = Task;
Task.PipeLine = Pipeline;
(Pipeline as ISubGridPipelineBase <TSubGridsRequestArgument>).CustomArgumentInitializer = CustomArgumentInitializer;
if ((Filters?.Filters?.Length ?? 0) == 0)
{
_log.LogError($"Filters supplied to pipeline processor is null or empty, replacing with single default filter");
Filters = new FilterSet(new CombinedFilter());
}
// Construct an aggregated set of excluded surveyed surfaces for the filters used in the query
foreach (var filter in Filters.Filters)
{
if (filter != null && SurveyedSurfaceExclusionList.Length > 0)
{
SurveyedSurfaceExclusionList = new Guid[filter.AttributeFilter.SurveyedSurfaceExclusionList.Length];
Array.Copy(filter.AttributeFilter.SurveyedSurfaceExclusionList, SurveyedSurfaceExclusionList,
SurveyedSurfaceExclusionList.Length);
}
}
// Get the SiteModel for the request
SiteModel = DIContext.Obtain <ISiteModels>().GetSiteModel(DataModelID);
if (SiteModel == null)
{
Response.ResultStatus = RequestErrorStatus.NoSuchDataModel;
return(false);
}
SpatialExtents = SiteModel.GetAdjustedDataModelSpatialExtents(SurveyedSurfaceExclusionList);
if (!SpatialExtents.IsValidPlanExtent)
{
Response.ResultStatus = RequestErrorStatus.FailedToRequestDatamodelStatistics; // Or there was no data in the model
return(false);
}
// Get the current production data existence map from the site model
ProdDataExistenceMap = SiteModel.ExistenceMap;
// Obtain the sub grid existence map for the project
// Retrieve the existence map for the datamodel
OverallExistenceMap = new SubGridTreeSubGridExistenceBitMask {
CellSize = SubGridTreeConsts.SubGridTreeDimension * SiteModel.CellSize
};
if (RequireSurveyedSurfaceInformation)
{
// Obtain local reference to surveyed surfaces (lock free access)
var localSurveyedSurfaces = SiteModel.SurveyedSurfaces;
if (localSurveyedSurfaces != null)
{
// Construct two filtered surveyed surface lists to act as a rolling pair used as arguments
// to the ProcessSurveyedSurfacesForFilter method
var filterSurveyedSurfaces = DIContext.Obtain <ISurveyedSurfaces>();
var filteredSurveyedSurfaces = DIContext.Obtain <ISurveyedSurfaces>();
SurveyedSurfacesExcludedViaTimeFiltering = Filters.Filters.Length > 0;
foreach (var filter in Filters.Filters)
{
if (!localSurveyedSurfaces.ProcessSurveyedSurfacesForFilter(DataModelID, filter,
filteredSurveyedSurfaces, filterSurveyedSurfaces, OverallExistenceMap))
{
Response.ResultStatus = RequestErrorStatus.FailedToRequestSubgridExistenceMap;
return(false);
}
SurveyedSurfacesExcludedViaTimeFiltering &= filterSurveyedSurfaces.Count == 0;
}
}
}
OverallExistenceMap.SetOp_OR(ProdDataExistenceMap);
foreach (var filter in Filters.Filters)
{
if (filter != null)
{
if (!DesignFilterUtilities.ProcessDesignElevationsForFilter(SiteModel, filter, OverallExistenceMap))
{
Response.ResultStatus = RequestErrorStatus.NoDesignProvided;
return(false);
}
Response.ResultStatus = FilterUtilities.PrepareFilterForUse(filter, DataModelID);
if (Response.ResultStatus != RequestErrorStatus.OK)
{
_log.LogInformation($"PrepareFilterForUse failed: Datamodel={DataModelID}");
return(false);
}
}
}
// Adjust the extents we have been given to encompass the spatial extent of the supplied filters (if any)
Filters.ApplyFilterAndSubsetBoundariesToExtents(SpatialExtents);
// If this request involves a relationship with a design then ensure the existence map
// for the design is loaded in to memory to allow the request pipeline to confine
// sub grid requests that overlay the actual design
if (RequestRequiresAccessToDesignFileExistenceMap)
{
if (CutFillDesign == null || CutFillDesign.DesignID == Guid.Empty)
{
_log.LogError($"No design provided to cut fill, summary volume or thickness overlay render request for datamodel {DataModelID}");
Response.ResultStatus = RequestErrorStatus.NoDesignProvided;
return(false);
}
DesignSubGridOverlayMap = GetExistenceMaps().GetSingleExistenceMap(DataModelID, Consts.EXISTENCE_MAP_DESIGN_DESCRIPTOR, CutFillDesign.DesignID);
if (DesignSubGridOverlayMap == null)
{
_log.LogError($"Failed to request sub grid overlay index for design {CutFillDesign.DesignID} in datamodel {DataModelID}");
Response.ResultStatus = RequestErrorStatus.NoDesignProvided;
return(false);
}
DesignSubGridOverlayMap.CellSize = SubGridTreeConsts.SubGridTreeDimension * SiteModel.CellSize;
}
// Impose the final restriction on the spatial extents from the client context
SpatialExtents.Intersect(OverrideSpatialExtents);
// Introduce the Request analyzer to the pipeline and spatial extents it requires
RequestAnalyser.Pipeline = Pipeline;
RequestAnalyser.WorldExtents = SpatialExtents;
ConfigurePipeline();
_log.LogInformation($"Pipeline processor build phase completed in {stopWatch.Elapsed}");
return(true);
}
catch (Exception e)
{
_log.LogError(e, "Exception occurred in pipeline builder");
throw;
}
}
public RequestErrorStatus ExecutePipeline()
{
SubGridPipelineAggregative <SubGridsRequestArgument, SimpleVolumesResponse> PipeLine;
var Result = RequestErrorStatus.Unknown;
var PipelineAborted = false;
// bool ShouldAbortDueToCompletedEventSet = false;
try
{
ProdDataExistenceMap = SiteModel.ExistenceMap;
try
{
if (ActiveDesign != null && (VolumeType == VolumeComputationType.BetweenFilterAndDesign || VolumeType == VolumeComputationType.BetweenDesignAndFilter))
{
if (ActiveDesign == null || ActiveDesign.Design.DesignDescriptor.IsNull)
{
Log.LogError($"No design provided to prod data/design volumes calc for datamodel {SiteModel.ID}");
return(RequestErrorStatus.NoDesignProvided);
}
DesignSubgridOverlayMap = GetExistenceMaps().GetSingleExistenceMap(SiteModel.ID, ExistenceMaps.Interfaces.Consts.EXISTENCE_MAP_DESIGN_DESCRIPTOR, ActiveDesign.Design.ID);
if (DesignSubgridOverlayMap == null)
{
return(RequestErrorStatus.NoDesignProvided);
}
}
OverallExistenceMap = new SubGridTreeSubGridExistenceBitMask();
// Work out the surveyed surfaces and coverage areas that need to be taken into account
var SurveyedSurfaces = SiteModel.SurveyedSurfaces;
if (SurveyedSurfaces != null)
{
// See if we need to handle surveyed surface data for 'base'
// Filter out any surveyed surfaces which don't match current filter (if any) - realistically, this is time filters we're thinking of here
if (VolumeType == VolumeComputationType.Between2Filters || VolumeType == VolumeComputationType.BetweenFilterAndDesign)
{
if (!SurveyedSurfaces.ProcessSurveyedSurfacesForFilter(SiteModel.ID, BaseFilter, FilteredTopSurveyedSurfaces, FilteredBaseSurveyedSurfaces, OverallExistenceMap))
{
return(RequestErrorStatus.Unknown);
}
}
// See if we need to handle surveyed surface data for 'top'
// Filter out any surveyed surfaces which don't match current filter (if any) - realistically, this is time filters we're thinking of here
if (VolumeType == VolumeComputationType.Between2Filters || VolumeType == VolumeComputationType.BetweenDesignAndFilter)
{
if (!SurveyedSurfaces.ProcessSurveyedSurfacesForFilter(SiteModel.ID, TopFilter, FilteredBaseSurveyedSurfaces, FilteredTopSurveyedSurfaces, OverallExistenceMap))
{
return(RequestErrorStatus.Unknown);
}
}
}
// Add in the production data existence map to the computed surveyed surfaces existence maps
OverallExistenceMap.SetOp_OR(ProdDataExistenceMap);
// If necessary, impose spatial constraints from filter design(s)
if (VolumeType == VolumeComputationType.Between2Filters || VolumeType == VolumeComputationType.BetweenFilterAndDesign)
{
if (!DesignFilterUtilities.ProcessDesignElevationsForFilter(SiteModel, BaseFilter, OverallExistenceMap))
{
return(RequestErrorStatus.Unknown);
}
}
if (VolumeType == VolumeComputationType.Between2Filters || VolumeType == VolumeComputationType.BetweenDesignAndFilter)
{
if (!DesignFilterUtilities.ProcessDesignElevationsForFilter(SiteModel, TopFilter, OverallExistenceMap))
{
return(RequestErrorStatus.Unknown);
}
}
var PipelinedTask = new VolumesComputationTask
{
Aggregator = Aggregator
};
try
{
PipeLine = new SubGridPipelineAggregative <SubGridsRequestArgument, SimpleVolumesResponse>(/*0, */ PipelinedTask);
PipelinedTask.PipeLine = PipeLine;
ConfigurePipeline(PipeLine);
if (PipeLine.Initiate())
{
var completionResult = PipeLine.WaitForCompletion();
Log.LogInformation(completionResult ? "WaitForCompletion successful" : $"WaitForCompletion timed out with {PipeLine.SubGridsRemainingToProcess} sub grids remaining to be processed");
if (PipeLine.SubGridsRemainingToProcess > 0)
{
Log.LogInformation($"Pipeline completed with {PipeLine.SubGridsRemainingToProcess} sub grids remaining to be processed");
}
}
/*
* while not FPipeLine.AllFinished and not FPipeLine.PipelineAborted do
* begin
* WaitResult := FPipeLine.CompleteEvent.WaitFor(5000);
*
* if VLPDSvcLocations.Debug_EmitSubgridPipelineProgressLogging then
* begin
* if ((FEpochCount > 0) or (FPipeLine.SubmissionNode.TotalNumberOfSubgridsScanned > 0)) and
* ((FPipeLine.OperationNode.NumPendingResultsReceived > 0) or (FPipeLine.OperationNode.OustandingSubgridsToOperateOn > 0)) then
* SIGLogMessage.PublishNoODS(Self, Format('%s: Pipeline (request %d, model %d): #Progress# - Scanned = %d, Submitted = %d, Processed = %d (with %d pending and %d results outstanding)',
* [Self.ClassName,
* FRequestDescriptor, FPipeline.ProjectUid,
* FPipeLine.SubmissionNode.TotalNumberOfSubgridsScanned,
* FPipeLine.SubmissionNode.TotalSumbittedSubgridRequests,
* FPipeLine.OperationNode.TotalOperatedOnSubgrids,
* FPipeLine.OperationNode.NumPendingResultsReceived,
* FPipeLine.OperationNode.OustandingSubgridsToOperateOn]), slmcDebug);
* end;
*
* if (WaitResult = wrSignaled) and not FPipeLine.AllFinished and not FPipeLine.PipelineAborted and not FPipeLine.Terminated then
* begin
* if ShouldAbortDueToCompletedEventSet then
* begin
* if (FPipeLine.OperationNode.NumPendingResultsReceived > 0) or (FPipeLine.OperationNode.OustandingSubgridsToOperateOn > 0) then
* SIGLogMessage.PublishNoODS(Self, Format('%s: Pipeline (request %d, model %d) being aborted as it''s completed event has remained set but still has work to do (%d outstanding subgrids, %d pending results to process) over a sleep epoch',
* [Self.ClassName,
* FRequestDescriptor, FPipeline.ProjectUid,
* FPipeLine.OperationNode.OustandingSubgridsToOperateOn,
* FPipeLine.OperationNode.NumPendingResultsReceived]), slmcError);
* FPipeLine.Abort;
* ASNodeImplInstance.AsyncResponder.ASNodeResponseProcessor.PerformTaskCancellation(FPipelinedTask);
* Exit;
* end
* else
* begin
* if (FPipeLine.OperationNode.NumPendingResultsReceived > 0) or (FPipeLine.OperationNode.OustandingSubgridsToOperateOn > 0) then
* SIGLogMessage.PublishNoODS(Self, Format('%s: Pipeline (request %d, model %d) has it''s completed event set but still has work to do (%d outstanding subgrids, %d pending results to process)',
* [Self.ClassName,
* FRequestDescriptor, FPipeline.ProjectUid,
* FPipeLine.OperationNode.OustandingSubgridsToOperateOn,
* FPipeLine.OperationNode.NumPendingResultsReceived]), slmcDebug);
* Sleep(500);
* ShouldAbortDueToCompletedEventSet := True;
* end;
* end;
*
* if FPipeLine.TimeToLiveExpired then
* begin
* FAbortedDueToTimeout := True;
* FPipeLine.Abort;
* ASNodeImplInstance.AsyncResponder.ASNodeResponseProcessor.PerformTaskCancellation(FPipelinedTask);
*
* // The pipeline has exceed its allotted time to complete. It will now
* // be aborted and this request will be failed.
* SIGLogMessage.PublishNoODS(Self, Format('%s: Pipeline (request %d) aborted due to time to live expiration (%d seconds)',
* [Self.ClassName, FRequestDescriptor, FPipeLine.TimeToLiveSeconds]), slmcError);
* Exit;
* end;
*/
PipelineAborted = PipeLine.Aborted;
if (!PipeLine.Terminated && !PipeLine.Aborted)
{
Result = RequestErrorStatus.OK;
}
}
finally
{
if (AbortedDueToTimeout)
{
Result = RequestErrorStatus.AbortedDueToPipelineTimeout;
}
else
if (PipelinedTask.IsCancelled || PipelineAborted)
{
Result = RequestErrorStatus.RequestHasBeenCancelled;
}
}
}
catch (Exception e)
{
Log.LogError(e, "ExecutePipeline raised exception");
}
return(Result);
}
catch (Exception e)
{
Log.LogError(e, "Exception");
}
return(RequestErrorStatus.Unknown);
}