/// <summary> /// Builds the pipeline configured per the supplied state ready to execute the request /// </summary> public bool Build() { // Todo: This method is left as async as a reminder that the GetExistenveMap workflows could either be async (as they // potentially read from the persistent store), and/or they couild be cached in the site model designs/surveyed surfaces contexts // See Jira CCSSSCON-1481 try { var stopWatch = Stopwatch.StartNew(); // Ensure the task is initialised with the request descriptor Task.RequestDescriptor = RequestDescriptor; // Ensure the Task grid data type matches the pipeline processor Task.GridDataType = GridDataType; // Introduce the task and the pipeline to each other Pipeline.PipelineTask = Task; Task.PipeLine = Pipeline; (Pipeline as ISubGridPipelineBase <TSubGridsRequestArgument>).CustomArgumentInitializer = CustomArgumentInitializer; if ((Filters?.Filters?.Length ?? 0) == 0) { _log.LogError($"Filters supplied to pipeline processor is null or empty, replacing with single default filter"); Filters = new FilterSet(new CombinedFilter()); } // Construct an aggregated set of excluded surveyed surfaces for the filters used in the query foreach (var filter in Filters.Filters) { if (filter != null && SurveyedSurfaceExclusionList.Length > 0) { SurveyedSurfaceExclusionList = new Guid[filter.AttributeFilter.SurveyedSurfaceExclusionList.Length]; Array.Copy(filter.AttributeFilter.SurveyedSurfaceExclusionList, SurveyedSurfaceExclusionList, SurveyedSurfaceExclusionList.Length); } } // Get the SiteModel for the request SiteModel = DIContext.Obtain <ISiteModels>().GetSiteModel(DataModelID); if (SiteModel == null) { Response.ResultStatus = RequestErrorStatus.NoSuchDataModel; return(false); } SpatialExtents = SiteModel.GetAdjustedDataModelSpatialExtents(SurveyedSurfaceExclusionList); if (!SpatialExtents.IsValidPlanExtent) { Response.ResultStatus = RequestErrorStatus.FailedToRequestDatamodelStatistics; // Or there was no data in the model return(false); } // Get the current production data existence map from the site model ProdDataExistenceMap = SiteModel.ExistenceMap; // Obtain the sub grid existence map for the project // Retrieve the existence map for the datamodel OverallExistenceMap = new SubGridTreeSubGridExistenceBitMask { CellSize = SubGridTreeConsts.SubGridTreeDimension * SiteModel.CellSize }; if (RequireSurveyedSurfaceInformation) { // Obtain local reference to surveyed surfaces (lock free access) var localSurveyedSurfaces = SiteModel.SurveyedSurfaces; if (localSurveyedSurfaces != null) { // Construct two filtered surveyed surface lists to act as a rolling pair used as arguments // to the ProcessSurveyedSurfacesForFilter method var filterSurveyedSurfaces = DIContext.Obtain <ISurveyedSurfaces>(); var filteredSurveyedSurfaces = DIContext.Obtain <ISurveyedSurfaces>(); SurveyedSurfacesExcludedViaTimeFiltering = Filters.Filters.Length > 0; foreach (var filter in Filters.Filters) { if (!localSurveyedSurfaces.ProcessSurveyedSurfacesForFilter(DataModelID, filter, filteredSurveyedSurfaces, filterSurveyedSurfaces, OverallExistenceMap)) { Response.ResultStatus = RequestErrorStatus.FailedToRequestSubgridExistenceMap; return(false); } SurveyedSurfacesExcludedViaTimeFiltering &= filterSurveyedSurfaces.Count == 0; } } } OverallExistenceMap.SetOp_OR(ProdDataExistenceMap); foreach (var filter in Filters.Filters) { if (filter != null) { if (!DesignFilterUtilities.ProcessDesignElevationsForFilter(SiteModel, filter, OverallExistenceMap)) { Response.ResultStatus = RequestErrorStatus.NoDesignProvided; return(false); } Response.ResultStatus = FilterUtilities.PrepareFilterForUse(filter, DataModelID); if (Response.ResultStatus != RequestErrorStatus.OK) { _log.LogInformation($"PrepareFilterForUse failed: Datamodel={DataModelID}"); return(false); } } } // Adjust the extents we have been given to encompass the spatial extent of the supplied filters (if any) Filters.ApplyFilterAndSubsetBoundariesToExtents(SpatialExtents); // If this request involves a relationship with a design then ensure the existence map // for the design is loaded in to memory to allow the request pipeline to confine // sub grid requests that overlay the actual design if (RequestRequiresAccessToDesignFileExistenceMap) { if (CutFillDesign == null || CutFillDesign.DesignID == Guid.Empty) { _log.LogError($"No design provided to cut fill, summary volume or thickness overlay render request for datamodel {DataModelID}"); Response.ResultStatus = RequestErrorStatus.NoDesignProvided; return(false); } DesignSubGridOverlayMap = GetExistenceMaps().GetSingleExistenceMap(DataModelID, Consts.EXISTENCE_MAP_DESIGN_DESCRIPTOR, CutFillDesign.DesignID); if (DesignSubGridOverlayMap == null) { _log.LogError($"Failed to request sub grid overlay index for design {CutFillDesign.DesignID} in datamodel {DataModelID}"); Response.ResultStatus = RequestErrorStatus.NoDesignProvided; return(false); } DesignSubGridOverlayMap.CellSize = SubGridTreeConsts.SubGridTreeDimension * SiteModel.CellSize; } // Impose the final restriction on the spatial extents from the client context SpatialExtents.Intersect(OverrideSpatialExtents); // Introduce the Request analyzer to the pipeline and spatial extents it requires RequestAnalyser.Pipeline = Pipeline; RequestAnalyser.WorldExtents = SpatialExtents; ConfigurePipeline(); _log.LogInformation($"Pipeline processor build phase completed in {stopWatch.Elapsed}"); return(true); } catch (Exception e) { _log.LogError(e, "Exception occurred in pipeline builder"); throw; } }
public RequestErrorStatus ExecutePipeline() { SubGridPipelineAggregative <SubGridsRequestArgument, SimpleVolumesResponse> PipeLine; var Result = RequestErrorStatus.Unknown; var PipelineAborted = false; // bool ShouldAbortDueToCompletedEventSet = false; try { ProdDataExistenceMap = SiteModel.ExistenceMap; try { if (ActiveDesign != null && (VolumeType == VolumeComputationType.BetweenFilterAndDesign || VolumeType == VolumeComputationType.BetweenDesignAndFilter)) { if (ActiveDesign == null || ActiveDesign.Design.DesignDescriptor.IsNull) { Log.LogError($"No design provided to prod data/design volumes calc for datamodel {SiteModel.ID}"); return(RequestErrorStatus.NoDesignProvided); } DesignSubgridOverlayMap = GetExistenceMaps().GetSingleExistenceMap(SiteModel.ID, ExistenceMaps.Interfaces.Consts.EXISTENCE_MAP_DESIGN_DESCRIPTOR, ActiveDesign.Design.ID); if (DesignSubgridOverlayMap == null) { return(RequestErrorStatus.NoDesignProvided); } } OverallExistenceMap = new SubGridTreeSubGridExistenceBitMask(); // Work out the surveyed surfaces and coverage areas that need to be taken into account var SurveyedSurfaces = SiteModel.SurveyedSurfaces; if (SurveyedSurfaces != null) { // See if we need to handle surveyed surface data for 'base' // Filter out any surveyed surfaces which don't match current filter (if any) - realistically, this is time filters we're thinking of here if (VolumeType == VolumeComputationType.Between2Filters || VolumeType == VolumeComputationType.BetweenFilterAndDesign) { if (!SurveyedSurfaces.ProcessSurveyedSurfacesForFilter(SiteModel.ID, BaseFilter, FilteredTopSurveyedSurfaces, FilteredBaseSurveyedSurfaces, OverallExistenceMap)) { return(RequestErrorStatus.Unknown); } } // See if we need to handle surveyed surface data for 'top' // Filter out any surveyed surfaces which don't match current filter (if any) - realistically, this is time filters we're thinking of here if (VolumeType == VolumeComputationType.Between2Filters || VolumeType == VolumeComputationType.BetweenDesignAndFilter) { if (!SurveyedSurfaces.ProcessSurveyedSurfacesForFilter(SiteModel.ID, TopFilter, FilteredBaseSurveyedSurfaces, FilteredTopSurveyedSurfaces, OverallExistenceMap)) { return(RequestErrorStatus.Unknown); } } } // Add in the production data existence map to the computed surveyed surfaces existence maps OverallExistenceMap.SetOp_OR(ProdDataExistenceMap); // If necessary, impose spatial constraints from filter design(s) if (VolumeType == VolumeComputationType.Between2Filters || VolumeType == VolumeComputationType.BetweenFilterAndDesign) { if (!DesignFilterUtilities.ProcessDesignElevationsForFilter(SiteModel, BaseFilter, OverallExistenceMap)) { return(RequestErrorStatus.Unknown); } } if (VolumeType == VolumeComputationType.Between2Filters || VolumeType == VolumeComputationType.BetweenDesignAndFilter) { if (!DesignFilterUtilities.ProcessDesignElevationsForFilter(SiteModel, TopFilter, OverallExistenceMap)) { return(RequestErrorStatus.Unknown); } } var PipelinedTask = new VolumesComputationTask { Aggregator = Aggregator }; try { PipeLine = new SubGridPipelineAggregative <SubGridsRequestArgument, SimpleVolumesResponse>(/*0, */ PipelinedTask); PipelinedTask.PipeLine = PipeLine; ConfigurePipeline(PipeLine); if (PipeLine.Initiate()) { var completionResult = PipeLine.WaitForCompletion(); Log.LogInformation(completionResult ? "WaitForCompletion successful" : $"WaitForCompletion timed out with {PipeLine.SubGridsRemainingToProcess} sub grids remaining to be processed"); if (PipeLine.SubGridsRemainingToProcess > 0) { Log.LogInformation($"Pipeline completed with {PipeLine.SubGridsRemainingToProcess} sub grids remaining to be processed"); } } /* * while not FPipeLine.AllFinished and not FPipeLine.PipelineAborted do * begin * WaitResult := FPipeLine.CompleteEvent.WaitFor(5000); * * if VLPDSvcLocations.Debug_EmitSubgridPipelineProgressLogging then * begin * if ((FEpochCount > 0) or (FPipeLine.SubmissionNode.TotalNumberOfSubgridsScanned > 0)) and * ((FPipeLine.OperationNode.NumPendingResultsReceived > 0) or (FPipeLine.OperationNode.OustandingSubgridsToOperateOn > 0)) then * SIGLogMessage.PublishNoODS(Self, Format('%s: Pipeline (request %d, model %d): #Progress# - Scanned = %d, Submitted = %d, Processed = %d (with %d pending and %d results outstanding)', * [Self.ClassName, * FRequestDescriptor, FPipeline.ProjectUid, * FPipeLine.SubmissionNode.TotalNumberOfSubgridsScanned, * FPipeLine.SubmissionNode.TotalSumbittedSubgridRequests, * FPipeLine.OperationNode.TotalOperatedOnSubgrids, * FPipeLine.OperationNode.NumPendingResultsReceived, * FPipeLine.OperationNode.OustandingSubgridsToOperateOn]), slmcDebug); * end; * * if (WaitResult = wrSignaled) and not FPipeLine.AllFinished and not FPipeLine.PipelineAborted and not FPipeLine.Terminated then * begin * if ShouldAbortDueToCompletedEventSet then * begin * if (FPipeLine.OperationNode.NumPendingResultsReceived > 0) or (FPipeLine.OperationNode.OustandingSubgridsToOperateOn > 0) then * SIGLogMessage.PublishNoODS(Self, Format('%s: Pipeline (request %d, model %d) being aborted as it''s completed event has remained set but still has work to do (%d outstanding subgrids, %d pending results to process) over a sleep epoch', * [Self.ClassName, * FRequestDescriptor, FPipeline.ProjectUid, * FPipeLine.OperationNode.OustandingSubgridsToOperateOn, * FPipeLine.OperationNode.NumPendingResultsReceived]), slmcError); * FPipeLine.Abort; * ASNodeImplInstance.AsyncResponder.ASNodeResponseProcessor.PerformTaskCancellation(FPipelinedTask); * Exit; * end * else * begin * if (FPipeLine.OperationNode.NumPendingResultsReceived > 0) or (FPipeLine.OperationNode.OustandingSubgridsToOperateOn > 0) then * SIGLogMessage.PublishNoODS(Self, Format('%s: Pipeline (request %d, model %d) has it''s completed event set but still has work to do (%d outstanding subgrids, %d pending results to process)', * [Self.ClassName, * FRequestDescriptor, FPipeline.ProjectUid, * FPipeLine.OperationNode.OustandingSubgridsToOperateOn, * FPipeLine.OperationNode.NumPendingResultsReceived]), slmcDebug); * Sleep(500); * ShouldAbortDueToCompletedEventSet := True; * end; * end; * * if FPipeLine.TimeToLiveExpired then * begin * FAbortedDueToTimeout := True; * FPipeLine.Abort; * ASNodeImplInstance.AsyncResponder.ASNodeResponseProcessor.PerformTaskCancellation(FPipelinedTask); * * // The pipeline has exceed its allotted time to complete. It will now * // be aborted and this request will be failed. * SIGLogMessage.PublishNoODS(Self, Format('%s: Pipeline (request %d) aborted due to time to live expiration (%d seconds)', * [Self.ClassName, FRequestDescriptor, FPipeLine.TimeToLiveSeconds]), slmcError); * Exit; * end; */ PipelineAborted = PipeLine.Aborted; if (!PipeLine.Terminated && !PipeLine.Aborted) { Result = RequestErrorStatus.OK; } } finally { if (AbortedDueToTimeout) { Result = RequestErrorStatus.AbortedDueToPipelineTimeout; } else if (PipelinedTask.IsCancelled || PipelineAborted) { Result = RequestErrorStatus.RequestHasBeenCancelled; } } } catch (Exception e) { Log.LogError(e, "ExecutePipeline raised exception"); } return(Result); } catch (Exception e) { Log.LogError(e, "Exception"); } return(RequestErrorStatus.Unknown); }