private void Test_GetCombinedExistenceMap_ViaDescriptor(SubGridTreeSubGridExistenceBitMask setBitMask1,
SubGridTreeSubGridExistenceBitMask setBitMask2)
{
var siteModel = DITAGFileAndSubGridRequestsWithIgniteFixture.NewEmptyModel();
var DesignUid1 = Guid.NewGuid();
var DesignUid2 = Guid.NewGuid();
var em = new TRex.ExistenceMaps.ExistenceMaps();
siteModel.PrimaryStorageProxy.WriteStreamToPersistentStore
(siteModel.ID,
BaseExistenceMapRequest.CacheKeyString(Consts.EXISTENCE_SURVEYED_SURFACE_DESCRIPTOR, DesignUid1),
FileSystemStreamType.DesignTopologyExistenceMap,
setBitMask1.ToStream(), setBitMask1.ToStream())
.Should().Be(FileSystemErrorStatus.OK);
siteModel.PrimaryStorageProxy.WriteStreamToPersistentStore
(siteModel.ID,
BaseExistenceMapRequest.CacheKeyString(Consts.EXISTENCE_SURVEYED_SURFACE_DESCRIPTOR, DesignUid2),
FileSystemStreamType.DesignTopologyExistenceMap,
setBitMask2.ToStream(), setBitMask2.ToStream())
.Should().Be(FileSystemErrorStatus.OK);
var getBitMask = em.GetCombinedExistenceMap(siteModel.ID, new Tuple <long, Guid>[]
{
new Tuple <long, Guid> (Consts.EXISTENCE_SURVEYED_SURFACE_DESCRIPTOR, DesignUid1),
new Tuple <long, Guid> (Consts.EXISTENCE_SURVEYED_SURFACE_DESCRIPTOR, DesignUid2)
});
setBitMask1.SetOp_OR(setBitMask2);
TestBitMasksAreTheSame(setBitMask1, getBitMask);
}
/// <summary>
/// Builds the pipeline configured per the supplied state ready to execute the request
/// </summary>
public bool Build()
{
// Todo: This method is left as async as a reminder that the GetExistenveMap workflows could either be async (as they
// potentially read from the persistent store), and/or they couild be cached in the site model designs/surveyed surfaces contexts
// See Jira CCSSSCON-1481
try
{
var stopWatch = Stopwatch.StartNew();
// Ensure the task is initialised with the request descriptor
Task.RequestDescriptor = RequestDescriptor;
// Ensure the Task grid data type matches the pipeline processor
Task.GridDataType = GridDataType;
// Introduce the task and the pipeline to each other
Pipeline.PipelineTask = Task;
Task.PipeLine = Pipeline;
(Pipeline as ISubGridPipelineBase <TSubGridsRequestArgument>).CustomArgumentInitializer = CustomArgumentInitializer;
if ((Filters?.Filters?.Length ?? 0) == 0)
{
_log.LogError($"Filters supplied to pipeline processor is null or empty, replacing with single default filter");
Filters = new FilterSet(new CombinedFilter());
}
// Construct an aggregated set of excluded surveyed surfaces for the filters used in the query
foreach (var filter in Filters.Filters)
{
if (filter != null && SurveyedSurfaceExclusionList.Length > 0)
{
SurveyedSurfaceExclusionList = new Guid[filter.AttributeFilter.SurveyedSurfaceExclusionList.Length];
Array.Copy(filter.AttributeFilter.SurveyedSurfaceExclusionList, SurveyedSurfaceExclusionList,
SurveyedSurfaceExclusionList.Length);
}
}
// Get the SiteModel for the request
SiteModel = DIContext.Obtain <ISiteModels>().GetSiteModel(DataModelID);
if (SiteModel == null)
{
Response.ResultStatus = RequestErrorStatus.NoSuchDataModel;
return(false);
}
SpatialExtents = SiteModel.GetAdjustedDataModelSpatialExtents(SurveyedSurfaceExclusionList);
if (!SpatialExtents.IsValidPlanExtent)
{
Response.ResultStatus = RequestErrorStatus.FailedToRequestDatamodelStatistics; // Or there was no data in the model
return(false);
}
// Get the current production data existence map from the site model
ProdDataExistenceMap = SiteModel.ExistenceMap;
// Obtain the sub grid existence map for the project
// Retrieve the existence map for the datamodel
OverallExistenceMap = new SubGridTreeSubGridExistenceBitMask {
CellSize = SubGridTreeConsts.SubGridTreeDimension * SiteModel.CellSize
};
if (RequireSurveyedSurfaceInformation)
{
// Obtain local reference to surveyed surfaces (lock free access)
var localSurveyedSurfaces = SiteModel.SurveyedSurfaces;
if (localSurveyedSurfaces != null)
{
// Construct two filtered surveyed surface lists to act as a rolling pair used as arguments
// to the ProcessSurveyedSurfacesForFilter method
var filterSurveyedSurfaces = DIContext.Obtain <ISurveyedSurfaces>();
var filteredSurveyedSurfaces = DIContext.Obtain <ISurveyedSurfaces>();
SurveyedSurfacesExcludedViaTimeFiltering = Filters.Filters.Length > 0;
foreach (var filter in Filters.Filters)
{
if (!localSurveyedSurfaces.ProcessSurveyedSurfacesForFilter(DataModelID, filter,
filteredSurveyedSurfaces, filterSurveyedSurfaces, OverallExistenceMap))
{
Response.ResultStatus = RequestErrorStatus.FailedToRequestSubgridExistenceMap;
return(false);
}
SurveyedSurfacesExcludedViaTimeFiltering &= filterSurveyedSurfaces.Count == 0;
}
}
}
OverallExistenceMap.SetOp_OR(ProdDataExistenceMap);
foreach (var filter in Filters.Filters)
{
if (filter != null)
{
if (!DesignFilterUtilities.ProcessDesignElevationsForFilter(SiteModel, filter, OverallExistenceMap))
{
Response.ResultStatus = RequestErrorStatus.NoDesignProvided;
return(false);
}
Response.ResultStatus = FilterUtilities.PrepareFilterForUse(filter, DataModelID);
if (Response.ResultStatus != RequestErrorStatus.OK)
{
_log.LogInformation($"PrepareFilterForUse failed: Datamodel={DataModelID}");
return(false);
}
}
}
// Adjust the extents we have been given to encompass the spatial extent of the supplied filters (if any)
Filters.ApplyFilterAndSubsetBoundariesToExtents(SpatialExtents);
// If this request involves a relationship with a design then ensure the existence map
// for the design is loaded in to memory to allow the request pipeline to confine
// sub grid requests that overlay the actual design
if (RequestRequiresAccessToDesignFileExistenceMap)
{
if (CutFillDesign == null || CutFillDesign.DesignID == Guid.Empty)
{
_log.LogError($"No design provided to cut fill, summary volume or thickness overlay render request for datamodel {DataModelID}");
Response.ResultStatus = RequestErrorStatus.NoDesignProvided;
return(false);
}
DesignSubGridOverlayMap = GetExistenceMaps().GetSingleExistenceMap(DataModelID, Consts.EXISTENCE_MAP_DESIGN_DESCRIPTOR, CutFillDesign.DesignID);
if (DesignSubGridOverlayMap == null)
{
_log.LogError($"Failed to request sub grid overlay index for design {CutFillDesign.DesignID} in datamodel {DataModelID}");
Response.ResultStatus = RequestErrorStatus.NoDesignProvided;
return(false);
}
DesignSubGridOverlayMap.CellSize = SubGridTreeConsts.SubGridTreeDimension * SiteModel.CellSize;
}
// Impose the final restriction on the spatial extents from the client context
SpatialExtents.Intersect(OverrideSpatialExtents);
// Introduce the Request analyzer to the pipeline and spatial extents it requires
RequestAnalyser.Pipeline = Pipeline;
RequestAnalyser.WorldExtents = SpatialExtents;
ConfigurePipeline();
_log.LogInformation($"Pipeline processor build phase completed in {stopWatch.Elapsed}");
return(true);
}
catch (Exception e)
{
_log.LogError(e, "Exception occurred in pipeline builder");
throw;
}
}
/// <summary>
/// Takes a site model spatial change map and incorporates those changes in the changes for each machine in the
/// site model.
/// Once items are processed they are removed from the change map queue retirement queue.
/// </summary>
private bool ProcessItem(ISiteModelChangeBufferQueueItem item)
{
try
{
if (item == null)
{
_log.LogError("Item supplied to queue processor is null. Aborting");
return(false);
}
if (item.Content == null)
{
_log.LogError("Item supplied to queue processor has no internal content. Aborting");
return(false);
}
var siteModel = DIContext.Obtain <ISiteModels>().GetSiteModel(item.ProjectUID);
if (siteModel == null)
{
_log.LogError($"Site model {item.ProjectUID} does not exist [deleted?]. Aborting");
return(false);
}
// Implement change map integration into machine change maps
// 0. Obtain transaction (will create implicit locks on items)
// 1. Read record for machine
// 2. Integrate new map
// 3. Write record back to store
// 4. Commit transaction
_log.LogInformation($"Processing an item: {item.Operation}, project:{item.ProjectUID}, machine:{item.MachineUid}");
var sw = Stopwatch.StartNew();
switch (item.Operation)
{
case SiteModelChangeMapOperation.AddSpatialChanges: // Add the two of them together...
{
// Add the spatial change to every machine in the site model
foreach (var machine in siteModel.Machines)
{
var l = _changeMapProxy.Lock(item.ProjectUID, machine.ID);
l.Enter();
try
{
var currentMask = _changeMapProxy.Get(item.ProjectUID, machine.ID);
if (currentMask == null)
{
currentMask = new SubGridTreeSubGridExistenceBitMask();
currentMask.SetOp_OR(siteModel.ExistenceMap);
}
// Extract the change map from the item
var updateMask = new SubGridTreeSubGridExistenceBitMask();
updateMask.FromBytes(item.Content);
currentMask.SetOp_OR(updateMask);
_changeMapProxy.Put(item.ProjectUID, machine.ID, currentMask);
}
finally
{
l.Exit();
}
}
break;
}
case SiteModelChangeMapOperation.RemoveSpatialChanges: // Subtract from the change map...
{
var l = _changeMapProxy.Lock(item.ProjectUID, item.MachineUid);
l.Enter();
try
{
// Remove the spatial change only from the machine the made the query
var currentMask = _changeMapProxy.Get(item.ProjectUID, item.MachineUid);
if (currentMask != null)
{
// Extract the change map from the item
var updateMask = new SubGridTreeSubGridExistenceBitMask();
currentMask.SetOp_ANDNOT(updateMask);
_changeMapProxy.Put(item.ProjectUID, item.MachineUid, currentMask);
}
}
finally
{
l.Exit();
}
break;
}
default:
_log.LogError($"Unknown operation encountered: {(int) item.Operation}");
return(false);
}
_log.LogInformation($"Completed processing an item in {sw.Elapsed}: {item.Operation}, project:{item.ProjectUID}, machine:{item.MachineUid}");
return(true);
}
catch (Exception e)
{
_log.LogError(e, "Exception thrown while processing queued items:");
throw;
}
}
private void CompareSiteModels(ISiteModel sm1, ISiteModel sm2,
int expectedExistanceMapSubGridCount, int expectedCallPassCount, int expectedNonNullCelCount)
{
var bitCount1 = sm1.ExistenceMap.CountBits();
var bitCount2 = sm2.ExistenceMap.CountBits();
// Check both site models have the same number of sub grids in their existence maps
bitCount1.Should().Be(bitCount2);
bitCount1.Should().Be(expectedExistanceMapSubGridCount);
// Check the content of the existence maps is identical
var testMap = new SubGridTreeSubGridExistenceBitMask();
testMap.SetOp_OR(sm1.ExistenceMap);
testMap.SetOp_XOR(sm2.ExistenceMap);
testMap.CountBits().Should().Be(0);
// The expected distribution of cell pass counts
long[] expectedCounts = { 93, 687, 68, 385, 57, 598, 65, 986, 52, 63, 0, 0, 0, 0, 0 };
// Scan the leaves in each model and count cell passes
int sm1Count = 0;
int sm1LeafCount = 0;
long[] actualCounts1 = new long[15];
int segmentCount1 = 0;
int nonNullCellCount1 = 0;
sm1.ExistenceMap.ScanAllSetBitsAsSubGridAddresses(address =>
{
sm1LeafCount++;
var leaf = SubGridUtilities.LocateSubGridContaining(sm1.PrimaryStorageProxy, sm1.Grid, address.X, address.Y, sm1.Grid.NumLevels, false, false) as IServerLeafSubGrid;
var iterator = new SubGridSegmentIterator(leaf, sm1.PrimaryStorageProxy);
while (iterator.MoveNext())
{
segmentCount1++;
TRex.SubGridTrees.Core.Utilities.SubGridUtilities.SubGridDimensionalIterator((x, y) =>
{
var passCount = iterator.CurrentSubGridSegment.PassesData.PassCount(x, y);
sm1Count += passCount;
if (passCount > 0)
{
nonNullCellCount1++;
actualCounts1[passCount - 1]++;
}
});
}
});
int sm2Count = 0;
int sm2LeafCount = 0;
long[] actualCounts2 = new long[15];
int segmentCount2 = 0;
int nonNullCellCount2 = 0;
sm2.ExistenceMap.ScanAllSetBitsAsSubGridAddresses(address =>
{
sm2LeafCount++;
var leaf = SubGridUtilities.LocateSubGridContaining(sm2.PrimaryStorageProxy, sm2.Grid, address.X, address.Y, sm2.Grid.NumLevels, false, false) as IServerLeafSubGrid;
var iterator = new SubGridSegmentIterator(leaf, sm2.PrimaryStorageProxy);
while (iterator.MoveNext())
{
segmentCount2++;
TRex.SubGridTrees.Core.Utilities.SubGridUtilities.SubGridDimensionalIterator((x, y) =>
{
var passCount = iterator.CurrentSubGridSegment.PassesData.PassCount(x, y);
sm2Count += passCount;
if (passCount > 0)
{
nonNullCellCount2++;
actualCounts2[passCount - 1]++;
}
});
}
});
segmentCount1.Should().Be(segmentCount2);
segmentCount1.Should().Be(sm1LeafCount);
sm1LeafCount.Should().Be(expectedExistanceMapSubGridCount);
sm1Count.Should().Be(sm2Count);
sm2LeafCount.Should().Be(expectedExistanceMapSubGridCount);
sm1Count.Should().Be(expectedCallPassCount);
actualCounts1.Should().BeEquivalentTo(actualCounts2);
actualCounts1.Should().BeEquivalentTo(expectedCounts);
nonNullCellCount1.Should().Be(nonNullCellCount2);
nonNullCellCount1.Should().Be(expectedNonNullCelCount);
}