public void Test_ElevationSubGridRequests_RequestElevationSubGrids_NoSurveyedSurfaces_NoFilter()
{
var siteModel = Utilities.ConstructModelForTestsWithTwoExcavatorMachineTAGFiles(out var processedTasks);
// Construct the set of requestors to query elevation sub grids needed for the summary volume calculations.
var utilities = DIContext.Obtain <IRequestorUtilities>();
var Requestors = utilities.ConstructRequestors(null, siteModel, new OverrideParameters(), new LiftParameters(),
utilities.ConstructRequestorIntermediaries(siteModel, new FilterSet(new CombinedFilter()), true, GridDataType.Height),
AreaControlSet.CreateAreaControlSet(), siteModel.ExistenceMap);
Requestors.Should().NotBeNull();
Requestors.Length.Should().Be(1);
// Request all elevation sub grids from the model
var requestedSubGrids = new List <IClientLeafSubGrid>();
siteModel.ExistenceMap.ScanAllSetBitsAsSubGridAddresses(x =>
{
var requestSubGridInternalResult = Requestors[0].RequestSubGridInternal(x, true, false);
if (requestSubGridInternalResult.requestResult == ServerRequestResult.NoError)
{
requestedSubGrids.Add(requestSubGridInternalResult.clientGrid);
}
});
requestedSubGrids.Count.Should().Be(4);
(requestedSubGrids[0] as IClientHeightLeafSubGrid).Cells[0, 0].Should().Be(Consts.NullHeight);
requestedSubGrids.Cast <IClientHeightLeafSubGrid>().Sum(x => x.CountNonNullCells()).Should().Be(427);
}
private List <IClientLeafSubGrid> GetSubGrids(CoordType coordType, OutputTypes outputType, bool isRawDataAsDBaseRequired,
out CSVExportRequestArgument requestArgument, out ISiteModel siteModel, string tagFileDirectory = "ElevationMappingMode-KettlewellDrive")
{
siteModel = SetupSiteAndRequestArgument(coordType, outputType, isRawDataAsDBaseRequired, tagFileDirectory, out requestArgument);
var overrides = requestArgument.Overrides;
var liftParams = requestArgument.LiftParams;
var utilities = DIContext.Obtain <IRequestorUtilities>();
var gridDataType = outputType == OutputTypes.PassCountLastPass || outputType == OutputTypes.VedaFinalPass
? GridDataType.CellProfile : GridDataType.CellPasses;
var requestors = utilities.ConstructRequestors(null, siteModel, requestArgument.Overrides, requestArgument.LiftParams,
utilities.ConstructRequestorIntermediaries(siteModel, requestArgument.Filters, false, gridDataType),
AreaControlSet.CreateAreaControlSet(), siteModel.ExistenceMap);
requestors.Should().NotBeNull();
requestors.Length.Should().Be(1);
// Request sub grids from the model
var requestedSubGrids = new List <IClientLeafSubGrid>();
siteModel.ExistenceMap.ScanAllSetBitsAsSubGridAddresses(x =>
{
var requestSubGridInternalResult = requestors[0].RequestSubGridInternal(x, true, false);
if (requestSubGridInternalResult.requestResult == ServerRequestResult.NoError)
{
requestedSubGrids.Add(requestSubGridInternalResult.clientGrid);
}
});
requestedSubGrids.Count.Should().Be(tagFileDirectory == "ElevationMappingMode-KettlewellDrive" ? 18 : 9);
return(requestedSubGrids);
}
public void MasksOutValues_WithoutSurveyedSurfaces()
{
SetupTestIgniteRouting();
var(siteModel, filter) = CreateSiteModelWithSimpleDesign();
// Create a sub grid at the Northwest origin so that it covers the small TIN design surrounding the
// [CellSize / 2, CellSize / 2] point
var baseTime = DateTime.UtcNow;
var cellPasses = new CellPass[32, 32][];
SubGridUtilities.SubGridDimensionalIterator((x, y) =>
{
cellPasses[x, y] = Enumerable.Range(0, 1).Select(p =>
new CellPass
{
Height = 1.0f,
InternalSiteModelMachineIndex = siteModel.Machines[0].InternalSiteModelMachineIndex,
Time = baseTime.AddMinutes(p),
PassType = PassType.Front
}).ToArray();
});
DITAGFileAndSubGridRequestsFixture.AddSingleSubGridWithPasses(siteModel, SubGridTreeConsts.DefaultIndexOriginOffset, SubGridTreeConsts.DefaultIndexOriginOffset, cellPasses);
// Construct a requestor and ask it to retrieve the sub grid from the site model, using the filter
// with the surface design mask
var utilities = DIContext.Obtain <IRequestorUtilities>();
var requestors = utilities.ConstructRequestors(null, siteModel,
new OverrideParameters(),
new LiftParameters(),
utilities.ConstructRequestorIntermediaries(siteModel,
new FilterSet(filter), false, GridDataType.Height),
AreaControlSet.CreateAreaControlSet(),
siteModel.ExistenceMap);
requestors.Length.Should().Be(1);
var response = requestors[0].RequestSubGridInternal
(new SubGridCellAddress(SubGridTreeConsts.DefaultIndexOriginOffset, SubGridTreeConsts.DefaultIndexOriginOffset),
true, false);
response.requestResult.Should().Be(ServerRequestResult.NoError);
response.clientGrid.Should().NotBeNull();
// Ensure the filtered cell has data
response.clientGrid.FilterMap[0, 0].Should().BeTrue();
(response.clientGrid as IClientHeightLeafSubGrid).Cells[0, 0].Should().Be(1.0f);
// Ensure no other cells have data
response.clientGrid.FilterMap.CountBits().Should().Be(1);
var subGrid = response.clientGrid as IClientHeightLeafSubGrid;
var count = 0;
subGrid.ForEach((x, y) => count += subGrid.Cells[x, y] == 1.0f ? 1 : 0);
count.Should().Be(1);
}
private ISubGridRequestor[] CreateRequestorsForSingleCellTesting(ISiteModel siteModel, GridDataType gridDataType, ICombinedFilter[] filters)
{
// Construct the set of requestors to query elevation sub grids needed for the summary volume calculations.
var utilities = DIContext.Obtain <IRequestorUtilities>();
var Requestors = utilities.ConstructRequestors(null, siteModel, new OverrideParameters(), new LiftParameters(),
utilities.ConstructRequestorIntermediaries(siteModel, new FilterSet(filters), true, gridDataType),
AreaControlSet.CreateAreaControlSet(), siteModel.ExistenceMap);
Requestors.Should().NotBeNull();
Requestors.Length.Should().Be(1);
return(Requestors);
}
/// <summary>
/// Gets the production data values for the requested cell
/// </summary>
private async Task GetProductionData(ISiteModel siteModel, IDesignWrapper cutFillDesign, CellDatumResponse_ClusterCompute result, CellDatumRequestArgument_ClusterCompute arg)
{
var existenceMap = siteModel.ExistenceMap;
var utilities = DIContext.Obtain <IRequestorUtilities>();
var requestors = utilities.ConstructRequestors(null, siteModel, arg.Overrides, arg.LiftParams,
utilities.ConstructRequestorIntermediaries(siteModel, arg.Filters, true, GridDataType.CellProfile),
AreaControlSet.CreateAreaControlSet(), existenceMap);
// Get the sub grid relative cell location
int cellX = arg.OTGCellX & SubGridTreeConsts.SubGridLocalKeyMask;
int cellY = arg.OTGCellY & SubGridTreeConsts.SubGridLocalKeyMask;
// Reach into the sub-grid request layer and retrieve an appropriate sub-grid
var cellOverrideMask = new SubGridTreeBitmapSubGridBits(SubGridBitsCreationOptions.Unfilled);
cellOverrideMask.SetBit(cellX, cellY);
requestors[0].CellOverrideMask = cellOverrideMask;
// using the cell address get the index of cell in clientGrid
var thisSubGridOrigin = new SubGridCellAddress(arg.OTGCellX, arg.OTGCellY);
var requestSubGridInternalResult = requestors[0].RequestSubGridInternal(thisSubGridOrigin, true, true);
if (requestSubGridInternalResult.requestResult != ServerRequestResult.NoError)
{
if (requestSubGridInternalResult.requestResult == ServerRequestResult.SubGridNotFound)
{
result.ReturnCode = CellDatumReturnCode.NoValueFound;
}
else
{
Log.LogError($"Request for sub grid {thisSubGridOrigin} request failed with code {requestSubGridInternalResult.requestResult}");
}
return;
}
var cell = ((ClientCellProfileLeafSubgrid)requestSubGridInternalResult.clientGrid).Cells[cellX, cellY];
if (cell.PassCount > 0) // Cell is not in our areaControlSet...
{
await ExtractRequiredValue(cutFillDesign, cell, result, arg);
result.TimeStampUTC = cell.LastPassTime;
}
}
public void SubGridForCaching_IgnoresFilterMask_WithPartialNonOverlappingOverrideMaskRetriction()
{
var siteModel = BuildModelForSubGridRequest();
var retriever = new SubGridRetriever(siteModel,
GridDataType.Height,
siteModel.PrimaryStorageProxy,
new CombinedFilter(),
new CellPassAttributeFilterProcessingAnnex(),
true, // Has override mask
BoundingIntegerExtent2D.Inverted(),
true, // prepareGridForCacheStorageIfNoSieving
1000,
AreaControlSet.CreateAreaControlSet(),
new FilteredValuePopulationControl(),
siteModel.ExistenceMap,
new OverrideParameters(),
new LiftParameters()
);
var clientGrid = ClientLeafSubGridFactoryFactory.CreateClientSubGridFactory().GetSubGridEx
(GridDataType.Height, SubGridTreeConsts.DefaultCellSize, SubGridTreeConsts.SubGridTreeLevels,
SubGridTreeConsts.DefaultIndexOriginOffset, SubGridTreeConsts.DefaultIndexOriginOffset);
var overrideMask = new SubGridTreeBitmapSubGridBits(SubGridBitsCreationOptions.Unfilled);
overrideMask[10, 10] = true; // This does not overlap the filter but should still return a result
var result = retriever.RetrieveSubGrid(clientGrid, overrideMask, out var seiveFilterInUse,
() =>
{
clientGrid.FilterMap.Clear();
clientGrid.FilterMap[0, 0] = true;
clientGrid.ProdDataMap.Fill();
return(ServerRequestResult.NoError);
});
result.Should().Be(ServerRequestResult.NoError);
seiveFilterInUse.Should().BeFalse();
clientGrid.FilterMap.CountBits().Should().Be(1); // Only asking for the one cell...
clientGrid.CountNonNullCells().Should().Be(1024); // All cells should be returned
}
/// <summary>
/// For each point in the list, get the sub grid and extract productionData at the station/offset i.e pointOfInterest
/// This could be optimized to get any poi from each sub grid before disposal
/// </summary>
private StationOffsetReportRequestResponse_ClusterCompute GetProductionData()
{
var result = new StationOffsetReportRequestResponse_ClusterCompute {
ResultStatus = RequestErrorStatus.Unknown
};
IDesignWrapper cutFillDesignWrapper = null;
if (requestArgument.ReferenceDesign != null && requestArgument.ReferenceDesign.DesignID != Guid.Empty)
{
var cutFillDesign = siteModel.Designs.Locate(requestArgument.ReferenceDesign.DesignID);
if (cutFillDesign == null)
{
throw new ArgumentException($"Design {requestArgument.ReferenceDesign.DesignID} not a recognized design in project {requestArgument.ProjectID}");
}
cutFillDesignWrapper = new DesignWrapper(requestArgument.ReferenceDesign, cutFillDesign);
}
var existenceMap = siteModel.ExistenceMap;
var utilities = DIContext.Obtain <IRequestorUtilities>();
var requestors = utilities.ConstructRequestors(null, siteModel, requestArgument.Overrides, requestArgument.LiftParams,
utilities.ConstructRequestorIntermediaries(siteModel, requestArgument.Filters, true, GridDataType.CellProfile),
AreaControlSet.CreateAreaControlSet(), existenceMap);
// Obtain the primary partition map to allow this request to determine the elements it needs to process
bool[] primaryPartitionMap = ImmutableSpatialAffinityPartitionMap.Instance().PrimaryPartitions();
SubGridTreeBitmapSubGridBits cellOverrideMask = new SubGridTreeBitmapSubGridBits(SubGridBitsCreationOptions.Unfilled);
foreach (var point in requestArgument.Points)
{
// Determine the on-the-ground cell
siteModel.Grid.CalculateIndexOfCellContainingPosition(point.Easting, point.Northing, out int OTGCellX, out int OTGCellY);
var thisSubGridOrigin = new SubGridCellAddress(OTGCellX, OTGCellY);
if (!primaryPartitionMap[thisSubGridOrigin.ToSpatialPartitionDescriptor()])
{
continue;
}
// Get the sub grid relative cell location
int cellX = OTGCellX & SubGridTreeConsts.SubGridLocalKeyMask;
int cellY = OTGCellY & SubGridTreeConsts.SubGridLocalKeyMask;
// Reach into the sub-grid request layer and retrieve an appropriate sub-grid
cellOverrideMask.Clear();
cellOverrideMask.SetBit(cellX, cellY);
requestors[0].CellOverrideMask = cellOverrideMask;
// using the cell address get the index of cell in clientGrid
var requestSubGridInternalResult = requestors[0].RequestSubGridInternal(
thisSubGridOrigin, true, true);
if (requestSubGridInternalResult.requestResult != ServerRequestResult.NoError)
{
Log.LogError($"Request for sub grid {thisSubGridOrigin} request failed with code {result}");
result.StationOffsetRows.Add(new StationOffsetRow(point.Station, point.Offset, point.Northing, point.Easting));
continue;
}
var hydratedPoint = ExtractRequiredValues(cutFillDesignWrapper, point, requestSubGridInternalResult.clientGrid as ClientCellProfileLeafSubgrid, cellX, cellY);
result.StationOffsetRows.Add(hydratedPoint);
}
result.ResultStatus = RequestErrorStatus.OK;
return(result);
}
/// <summary>
/// Executor that implements requesting and rendering sub grid information to create the cell datum
/// </summary>
public async Task <CellPassesResponse> ExecuteAsync(CellPassesRequestArgument_ClusterCompute arg, SubGridSpatialAffinityKey key)
{
Log.LogInformation($"Performing Execute for DataModel:{arg.ProjectID}");
var result = new CellPassesResponse {
ReturnCode = CellPassesReturnCode.Error
};
var siteModel = DIContext.Obtain <ISiteModels>().GetSiteModel(arg.ProjectID);
if (siteModel == null)
{
Log.LogError($"Failed to locate site model {arg.ProjectID}");
return(result);
}
var existenceMap = siteModel.ExistenceMap;
var utilities = DIContext.Obtain <IRequestorUtilities>();
var requestors = utilities.ConstructRequestors(null, siteModel, arg.Overrides, arg.LiftParams,
utilities.ConstructRequestorIntermediaries(siteModel, arg.Filters, true, GridDataType.CellPasses),
AreaControlSet.CreateAreaControlSet(),
existenceMap);
// Get the sub grid relative cell location
var cellX = arg.OTGCellX & SubGridTreeConsts.SubGridLocalKeyMask;
var cellY = arg.OTGCellY & SubGridTreeConsts.SubGridLocalKeyMask;
// Reach into the sub-grid request layer and retrieve an appropriate sub-grid
var cellOverrideMask = new SubGridTreeBitmapSubGridBits(SubGridBitsCreationOptions.Unfilled);
cellOverrideMask.SetBit(cellX, cellY);
requestors[0].CellOverrideMask = cellOverrideMask;
var thisSubGridOrigin = new SubGridCellAddress(arg.OTGCellX, arg.OTGCellY);
var requestSubGridInternalResult = requestors[0].RequestSubGridInternal(thisSubGridOrigin, true, true);
if (requestSubGridInternalResult.requestResult != ServerRequestResult.NoError)
{
if (requestSubGridInternalResult.requestResult == ServerRequestResult.SubGridNotFound)
{
result.ReturnCode = CellPassesReturnCode.NoDataFound;
}
else
{
Log.LogError($"Request for sub grid {thisSubGridOrigin} request failed with code {requestSubGridInternalResult.requestResult}");
}
return(result);
}
if (!(requestSubGridInternalResult.clientGrid is ClientCellProfileAllPassesLeafSubgrid grid))
{
Log.LogError($"Request for sub grid {thisSubGridOrigin} request failed due the grid return type being incorrect. Expected {typeof(ClientCellProfileAllPassesLeafSubgrid).Name}, but got {requestSubGridInternalResult.clientGrid.GetType().Name}");
return(result);
}
var cell = grid.Cells[cellX, cellY];
if (cell.TotalPasses > 0)
{
result.ReturnCode = CellPassesReturnCode.DataFound;
for (var idx = 0; idx < cell.TotalPasses; idx++)
{
var cellPass = cell.CellPasses[idx];
result.CellPasses.Add(cellPass);
}
}
else
{
result.ReturnCode = CellPassesReturnCode.NoDataFound;
}
return(result);
}
/// <summary>
/// Builds a fully analyzed vector of profiled cells from the list of cell passed to it
/// </summary>
public override bool Analyze(List <SummaryVolumeProfileCell> profileCells, ISubGridSegmentCellPassIterator cellPassIterator)
{
Log.LogDebug($"Analyze Summary Volume ProfileCells. Processing {profileCells.Count}");
var CurrentSubgridOrigin = new SubGridCellAddress(int.MaxValue, int.MaxValue);
ISubGrid SubGrid = null;
IServerLeafSubGrid _SubGridAsLeaf = null;
profileCell = null;
// Construct the set of requestors to query elevation sub grids needed for the summary volume calculations.
var filterSet = FilterUtilities.ConstructFilters(FilterSet, VolumeType);
IntermediaryFilterRequired = filterSet.Filters.Length == 3;
var utilities = DIContext.Obtain <IRequestorUtilities>();
Requestors = utilities.ConstructRequestors(null, SiteModel, Overrides, LiftParams,
utilities.ConstructRequestorIntermediaries(SiteModel, filterSet, true, GridDataType.HeightAndTime),
AreaControlSet.CreateAreaControlSet(), PDExistenceMap);
var cellOverrideMask = new SubGridTreeBitmapSubGridBits(SubGridBitsCreationOptions.Unfilled);
for (int I = 0; I < profileCells.Count; I++)
{
profileCell = profileCells[I];
// get sub grid origin for cell address
var thisSubgridOrigin = new SubGridCellAddress(profileCell.OTGCellX >> SubGridTreeConsts.SubGridIndexBitsPerLevel,
profileCell.OTGCellY >> SubGridTreeConsts.SubGridIndexBitsPerLevel);
if (!CurrentSubgridOrigin.Equals(thisSubgridOrigin)) // if we have a new sub grid to fetch
{
// if we have an existing sub grid and a change in sub grid detected process the current sub grid profile cell list
if (SubGrid != null)
{
ProcessSubGroup(new SubGridCellAddress(CurrentSubgridOrigin.X << SubGridTreeConsts.SubGridIndexBitsPerLevel, CurrentSubgridOrigin.Y << SubGridTreeConsts.SubGridIndexBitsPerLevel),
PDExistenceMap[CurrentSubgridOrigin.X, CurrentSubgridOrigin.Y], cellOverrideMask);
cellOverrideMask.Clear();
}
SubGrid = null;
cellCounter = 0;
// Does the sub grid tree contain this node in it's existence map? if so get sub grid
if (PDExistenceMap[thisSubgridOrigin.X, thisSubgridOrigin.Y])
{
SubGrid = SubGridTrees.Server.Utilities.SubGridUtilities.LocateSubGridContaining
(SiteModel.PrimaryStorageProxy, SiteModel.Grid, profileCell.OTGCellX, profileCell.OTGCellY, SiteModel.Grid.NumLevels, false, false);
}
_SubGridAsLeaf = SubGrid as ServerSubGridTreeLeaf;
if (_SubGridAsLeaf == null)
{
continue;
}
CurrentSubgridOrigin = thisSubgridOrigin; // all good to proceed with this sub grid
}
profileCellList[cellCounter++] = profileCell; // add cell to list to process for this sub grid
cellOverrideMask.SetBit(profileCell.OTGCellX & SubGridTreeConsts.SubGridLocalKeyMask, profileCell.OTGCellY & SubGridTreeConsts.SubGridLocalKeyMask);
}
if (cellCounter > 0 && SubGrid != null) // Make sure we process last list
{
ProcessSubGroup(new SubGridCellAddress(CurrentSubgridOrigin.X << SubGridTreeConsts.SubGridIndexBitsPerLevel, CurrentSubgridOrigin.Y << SubGridTreeConsts.SubGridIndexBitsPerLevel),
PDExistenceMap[CurrentSubgridOrigin.X, CurrentSubgridOrigin.Y], cellOverrideMask);
}
return(true);
}
public void Test_RequestorUtilities_CreateRequestors_DefaultFilters(int filterCount)
{
var ru = new RequestorUtilities();
ISurveyedSurfaces surveyedSurfaces = DIContext.Obtain <ISurveyedSurfaces>();
// Create two surveyed surfaces that bracket current time by one day either side and set the filter end time to be current time
// which will cause only one surveyed surface to be filtered
Guid ssGuid1 = Guid.NewGuid();
var ss1 = surveyedSurfaces.AddSurveyedSurfaceDetails(ssGuid1, DesignDescriptor.Null(), Consts.MIN_DATETIME_AS_UTC, BoundingWorldExtent3D.Null());
var mockGrid = new Mock <IServerSubGridTree>();
mockGrid.Setup(x => x.CellSize).Returns(SubGridTreeConsts.DefaultCellSize);
var mockSiteModel = new Mock <ISiteModel>();
mockSiteModel.Setup(x => x.SurveyedSurfacesLoaded).Returns(true);
mockSiteModel.Setup(x => x.SurveyedSurfaces).Returns(surveyedSurfaces);
mockSiteModel.Setup(x => x.Grid).Returns(mockGrid.Object);
ICombinedFilter[] filters = Enumerable.Range(1, filterCount).Select(x => new CombinedFilter()).ToArray();
IFilterSet filterSet = new FilterSet(filters);
var intermediaries = ru.ConstructRequestorIntermediaries(mockSiteModel.Object, filterSet, true, GridDataType.Height);
var requestors = ru.ConstructRequestors(new SubGridsRequestArgument(), mockSiteModel.Object, new OverrideParameters(), new LiftParameters(), intermediaries, AreaControlSet.CreateAreaControlSet(), null);
requestors.Length.Should().Be(filters.Length);
for (var i = 0; i < requestors.Length; i++)
{
requestors[i].CellOverrideMask.Should().NotBe(null);
}
}