public void MetaTest_Test_MakeSubgridWith10240CellPassesAtOneSecondIntervals()
{
// Create a subgrid to hold the segment
IServerLeafSubGrid subGrid = MakeSubgridWith10240CellPassesAtOneSecondIntervals();
ISubGridCellPassesDataSegment segment = subGrid.Cells.PassesData[0];
// Check all cells have exactly 10 passes
VSS.TRex.SubGridTrees.Core.Utilities.SubGridUtilities.SubGridDimensionalIterator((x, y) =>
{
Assert.True(segment.PassesData.PassCount(x, y) == 10,
$"Cell in segment at {x},{y} does not have 10 cell passes");
});
//Check the total number of passes is 10240, and the maximum pass count is 10
segment.PassesData.CalculateTotalPasses(out int totalPassCount, out _, out int maximumPassCount);
Assert.True(10240 == totalPassCount, "Initial total pass count not 10240");
Assert.True(10240 == segment.PassesData.SegmentPassCount,
$"segment.PassesData.SegmentPassCount does not equal 10240 (it is {segment.PassesData.SegmentPassCount})");
Assert.True(10 == maximumPassCount, "Initial maximum pass count not 10");
// Check the time range is as expected
segment.PassesData.CalculateTimeRange(out DateTime startSegmentTime, out DateTime endSegmentTime);
Assert.True(endSegmentTime > startSegmentTime, $"End time {endSegmentTime} not greater than startTime {startSegmentTime}");
Assert.True(startSegmentTime == startTime, $"Start time {startSegmentTime} not equal to {startTime} as expected");
Assert.True(endSegmentTime == startTime.AddSeconds(10239), $"End time {endSegmentTime} not equal to {startTime.AddSeconds(10239)} as expected");
}
private void IntegrateIntoLiveGrid(IServerLeafSubGrid sourceSubGrid,
ISubGridSegmentIterator segmentIterator,
Action <int, int> subGridChangeNotifier)
{
var targetSubGrid = LocateOrCreateSubGrid(_target, sourceSubGrid.OriginX, sourceSubGrid.OriginY);
try
{
if (targetSubGrid == null)
{
_log.LogError("Failed to locate or create sub grid in IntegrateIntoLiveGrid");
return;
}
if (!IntegrateIntoLiveDatabase(sourceSubGrid, targetSubGrid, segmentIterator, subGridChangeNotifier))
{
_log.LogError("Integration into live database failed");
}
}
finally
{
// At this point TargetSubGrid is of no further interest. The site model will be dropped in-toto once all
// changes have been integrated into the live sub grids and releasing it's resources here alleviates memory
// pressure that occurs if all sub grids are integrated before any resource freeing occurs
// This is not mediated by a using block as we want the sub grid to remain in the sub grid tree (ie:
// removing it is messy and not necessary here, it can be delegated to the DropSiteModel() phase.
targetSubGrid?.DeAllocateLeafFullPassStacks();
targetSubGrid?.DeAllocateLeafLatestPassGrid();
}
}
public ISubGridCellPassesDataSegment AddNewSegment(IServerLeafSubGrid subGrid,
ISubGridCellPassesDataSegmentInfo segmentInfo)
{
if (segmentInfo == null)
{
throw new TRexSubGridProcessingException($"Null segment info passed to AddNewSegment for sub grid {subGrid.Moniker()}");
}
if (segmentInfo.Segment != null)
{
throw new TRexSubGridProcessingException($"Segment info passed to AddNewSegment for sub grid {subGrid.Moniker()} already contains an allocated segment");
}
var Result = new SubGridCellPassesDataSegment
{
Owner = subGrid,
SegmentInfo = segmentInfo
};
segmentInfo.Segment = Result;
for (int I = 0; I < Count; I++)
{
if (segmentInfo.EndTime <= Items[I].SegmentInfo.StartTime)
{
Items.Insert(I, Result);
if (_performSegmentAdditionIntegrityChecks)
{
for (int J = 0; J < Count - 1; J++)
{
if (Items[J].SegmentInfo.StartTime >= Items[J + 1].SegmentInfo.StartTime)
{
_log.LogError($"Segment passes list out of order {Items[J].SegmentInfo.StartTime} versus {Items[J + 1].SegmentInfo.StartTime}. Segment count = {Count}");
DumpSegmentsToLog();
throw new TRexSubGridProcessingException($"Segment passes list out of order {Items[J].SegmentInfo.StartTime} versus {Items[J + 1].SegmentInfo.StartTime}. Segment count = {Count}");
}
}
}
return(Result);
}
}
// if we get to here, then the new segment is at the end of the list, so just add it to the end
Add(Result);
return(Result);
}
private bool IntegrateIntoLiveDatabase(IServerLeafSubGrid sourceSubGrid,
IServerLeafSubGrid targetSubGrid,
ISubGridSegmentIterator segmentIterator,
Action <int, int> subGridChangeNotifier)
{
// Note the fact that this sub grid will be changed and become dirty as a result
// of the cell pass integration
targetSubGrid.SetDirty();
targetSubGrid.Integrate(sourceSubGrid, segmentIterator, false);
subGridChangeNotifier?.Invoke(targetSubGrid.OriginX, targetSubGrid.OriginY);
// Save the integrated state of the sub grid segments to allow Ignite to store & socialize the update
// within the cluster.
// Failure to save a piece of data aborts the entire integration
var result = false;
if (_target.SaveLeafSubGrid(targetSubGrid, _storageProxySubGrids, _storageProxySubGridSegments, InvalidatedSpatialStreams))
{
// Successfully saving the sub grid directory information is the point at which this sub grid may be recognized to exist
// in the site model. Note this by including it within the SiteModel existence map
_siteModel.ExistenceMap.SetCell(targetSubGrid.OriginX >> SubGridTreeConsts.SubGridIndexBitsPerLevel,
targetSubGrid.OriginY >> SubGridTreeConsts.SubGridIndexBitsPerLevel,
true);
result = true;
}
else
{
_log.LogError($"Sub grid leaf save failed for {targetSubGrid}, existence map not modified.");
}
// Finally, mark the source sub grid as not being dirty. We need to do this to allow
// the sub grid to permit its destruction as all changes have been merged into the target.
if (result)
{
sourceSubGrid.AllChangesMigrated();
}
return(result);
}
private void IntegrateSubGrid(IServerLeafSubGrid sourceSubGrid,
SubGridTreeIntegrationMode integrationMode,
Action <int, int> subGridChangeNotifier,
ISubGridSegmentIterator segmentIterator)
{
// Locate a matching sub grid in this tree. If there is none, then create it
// and assign the sub grid from the iterator to it. If there is one, process
// the cell pass stacks merging the two together
var integratingIntoIntermediaryGrid = integrationMode == SubGridTreeIntegrationMode.UsingInMemoryTarget;
if (integratingIntoIntermediaryGrid)
{
IntegrateIntoIntermediaryGrid(sourceSubGrid, segmentIterator);
}
else
{
IntegrateIntoLiveGrid(sourceSubGrid, segmentIterator, subGridChangeNotifier);
}
}
public void Test_SubgridSegment_Cleaver()
{
// Create a sub grid to hold the segment
IServerLeafSubGrid subGrid = MakeSubgridWith10240CellPassesAtOneSecondIntervals();
// Exercise the cleaver!
// Instruct the segment container to cleave the segment
// Set the segment to not dirty - it should be ignored
subGrid.Cells.PassesData[0].Dirty = false;
var cleaver = new SubGridSegmentCleaver();
cleaver.PerformSegmentCleaving(StorageProxy.Instance(StorageMutability.Mutable), subGrid, 10000);
Assert.True(1 == subGrid.Cells.PassesData.Count, $"After cleaving with no dirty segments there are {subGrid.Cells.PassesData.Count} segments instead of the expected one segments");
// Set the segment to not dirty - it should be ignored
subGrid.Cells.PassesData[0].Dirty = true;
cleaver = new SubGridSegmentCleaver();
cleaver.PerformSegmentCleaving(StorageProxy.Instance(StorageMutability.Mutable), subGrid, 10000);
//Check there are now two segments in total
Assert.True(2 == subGrid.Cells.PassesData.Count, $"After cleaving there are {subGrid.Cells.PassesData.Count} segments instead of the expected two segments");
//Check the total number of passes across the two segments is 10240, and the maximum pass count is 5
ISubGridCellPassesDataSegment segment1 = subGrid.Cells.PassesData[0];
ISubGridCellPassesDataSegment segment2 = subGrid.Cells.PassesData[1];
segment1.PassesData.CalculateTotalPasses(out int totalPassCount1, out _, out int maximumPassCount1);
segment2.PassesData.CalculateTotalPasses(out int totalPassCount2, out _, out int maximumPassCount2);
Assert.True(10240 == (totalPassCount1 + totalPassCount2), $"Totals ({totalPassCount1} and {totalPassCount2} don't add up to 10240 after cleaving");
Assert.True(5 == maximumPassCount1, $"Maximum pass count 1 {maximumPassCount1}, is not 5");
Assert.True(5 == maximumPassCount2, $"Maximum pass count 2 {maximumPassCount2}, is not 5");
// Check the segment pass count in the segment is correct
Assert.True(totalPassCount1 == segment1.PassesData.SegmentPassCount, $"Total passes for segment 1 {totalPassCount1} is not equal to segmentPassCount in that segment {segment1.PassesData.SegmentPassCount}");
Assert.True(totalPassCount2 == segment2.PassesData.SegmentPassCount, $"Total passes for segment 2 {totalPassCount2} is not equal to segmentPassCount in that segment {segment2.PassesData.SegmentPassCount}");
}
private void IntegrateIntoIntermediaryGrid(IServerLeafSubGrid sourceSubGrid, ISubGridSegmentIterator segmentIterator)
{
var targetSubGrid = _target.ConstructPathToCell(sourceSubGrid.OriginX, sourceSubGrid.OriginY,
SubGridPathConstructionType.CreateLeaf) as IServerLeafSubGrid;
targetSubGrid.AllocateLeafFullPassStacks();
// If the node is brand new (ie: it does not have any cell passes committed to it yet)
// then create and select the default segment
if (targetSubGrid.Directory.SegmentDirectory.Count == 0)
{
targetSubGrid.Cells.SelectSegment(Consts.MIN_DATETIME_AS_UTC);
targetSubGrid.Cells.PassesData[0].AllocateFullPassStacks();
}
if (targetSubGrid.Cells.PassesData[0].PassesData == null)
{
_log.LogCritical("No segment passes data in new segment");
return;
}
targetSubGrid.Integrate(sourceSubGrid, segmentIterator, true);
}
public void Test_SubgridSegmentCleaving()
{
// Create a sub grid to hold the segment. the subGrid retains the limit in the subGrid constructor in MakeSubgridWith10240CellPassesAtOneSecondIntervals()
IServerLeafSubGrid subGrid = MakeSubgridWith10240CellPassesAtOneSecondIntervals();
ISubGridCellPassesDataSegment segment = subGrid.Cells.PassesData[0];
// Instruct the segment container to cleave the segment with a limit of 100000
var newSegmentsFromCleaving = new List <ISubGridCellPassesDataSegment>();
var persistedClovenSegments = new List <ISubGridSpatialAffinityKey>();
Assert.False(subGrid.Cells.CleaveSegment(segment, newSegmentsFromCleaving, persistedClovenSegments, 100000), "Segment was cloven when cell pass count was below limit");
// Set the cleaving limit to 10000 to force the segment TO BE cloven = the cleave result should be true
subGrid = MakeSubgridWith10240CellPassesAtOneSecondIntervals();
segment = subGrid.Cells.PassesData[0];
newSegmentsFromCleaving = new List <ISubGridCellPassesDataSegment>();
persistedClovenSegments = new List <ISubGridSpatialAffinityKey>();
Assert.True(subGrid.Cells.CleaveSegment(segment, newSegmentsFromCleaving, persistedClovenSegments, 10000), "Segment failed to cleave with pass count above limit");
//Check there are now two segments in total
Assert.True(2 == subGrid.Cells.PassesData.Count, $"After cleaving there are {subGrid.Cells.PassesData.Count} segments instead of the expected two segments");
//Check the total number of passes across the two segments is 10240, and the maximum pass count is 5
ISubGridCellPassesDataSegment segment1 = subGrid.Cells.PassesData[0];
ISubGridCellPassesDataSegment segment2 = subGrid.Cells.PassesData[1];
segment1.PassesData.CalculateTotalPasses(out int totalPassCount1, out _, out int maximumPassCount1);
segment2.PassesData.CalculateTotalPasses(out int totalPassCount2, out _, out int maximumPassCount2);
Assert.True(10240 == totalPassCount1 + totalPassCount2, $"Totals ({totalPassCount1} and {totalPassCount2} don't add up to 10240 after cleaving");
Assert.True(5 == maximumPassCount1, $"Maximum pass count 1 {maximumPassCount1}, is not 5");
Assert.True(5 == maximumPassCount2, $"Maximum pass count 2 {maximumPassCount2}, is not 5");
// Check the segment pass count in the segment is correct
Assert.True(totalPassCount1 == segment1.PassesData.SegmentPassCount, $"Total passes for segment 1 {totalPassCount1} is not equal to segmentPassCount in that segment {segment1.PassesData.SegmentPassCount}");
Assert.True(totalPassCount2 == segment2.PassesData.SegmentPassCount, $"Total passes for segment 2 {totalPassCount2} is not equal to segmentPassCount in that segment {segment2.PassesData.SegmentPassCount}");
}
public static void AddMultipleCellsWithPasses(ISiteModel siteModel, int cellX, int cellY,
List <CellPass[]> passesList, int expectedCellCount = -1, int expectedPassCount = -1)
{
// Construct the sub grid to hold the cell being tested
IServerLeafSubGrid leaf = siteModel.Grid.ConstructPathToCell(cellX, cellY, SubGridPathConstructionType.CreateLeaf) as IServerLeafSubGrid;
leaf.Should().NotBeNull();
leaf.AllocateLeafFullPassStacks();
leaf.CreateDefaultSegment();
leaf.AllocateFullPassStacks(leaf.Directory.SegmentDirectory.First());
leaf.AllocateLeafLatestPassGrid();
// Add the leaf to the site model existence map
siteModel.ExistenceMap[leaf.OriginX >> SubGridTreeConsts.SubGridIndexBitsPerLevel, leaf.OriginY >> SubGridTreeConsts.SubGridIndexBitsPerLevel] = true;
long totalCells = 0;
for (var i = 0; i < passesList.Count; i++)
{
//CellPass[] _passes = passes.ToArray();
byte subGridX = (byte)(cellX & SubGridTreeConsts.SubGridLocalKeyMask);
byte subGridY = (byte)(cellY & SubGridTreeConsts.SubGridLocalKeyMask);
foreach (var pass in passesList[i])
{
leaf.AddPass(subGridX, subGridY, pass);
}
var cellPasses = leaf.Cells.PassesData[i].PassesData.ExtractCellPasses(subGridX, subGridY);
if (expectedPassCount > -1)
{
((int)cellPasses.PassCount).Should().Be(expectedPassCount);
}
// Assign global latest cell pass to the appropriate pass
leaf.Directory.GlobalLatestCells[subGridX, subGridY] = cellPasses.Passes.Last();
// Ensure the pass data existence map records the existence of a non null value in the cell
leaf.Directory.GlobalLatestCells.PassDataExistenceMap[subGridX, subGridY] = true;
if (expectedCellCount > -1)
{
// Count the number of non-null elevation cells to verify a correct setup
siteModel.Grid.Root.ScanSubGrids(siteModel.Grid.FullCellExtent(), x =>
{
totalCells += leaf.Directory.GlobalLatestCells.PassDataExistenceMap.CountBits();
return(true);
});
}
siteModel.SiteModelExtent.Include(siteModel.Grid.GetCellExtents(cellX, cellY));
}
totalCells.Should().Be(expectedCellCount);
// Save the leaf information just created
siteModel.Grid.SaveLeafSubGrid(leaf, siteModel.PrimaryStorageProxy, siteModel.PrimaryStorageProxy, new List <ISubGridSpatialAffinityKey>());
siteModel.SaveToPersistentStoreForTAGFileIngest(siteModel.PrimaryStorageProxy);
}
/// <summary>
/// Cleaves all dirty segments requiring cleaving within the given sub grid
/// </summary>
public void PerformSegmentCleaving(IStorageProxy storageProxyForSubGridSegments, IServerLeafSubGrid subGrid, int subGridSegmentPassCountLimit = 0)
{
var iterator = new SubGridSegmentIterator(subGrid, storageProxyForSubGridSegments)
{
IterationDirection = IterationDirection.Forwards,
ReturnDirtyOnly = true,
RetrieveAllPasses = true
};
var origin = new SubGridCellAddress(subGrid.OriginX, subGrid.OriginY);
if (!iterator.MoveToFirstSubGridSegment())
{
return;
}
do
{
var segment = iterator.CurrentSubGridSegment;
var cleavedTimeRangeStart = segment.SegmentInfo.StartTime;
var cleavedTimeRangeEnd = segment.SegmentInfo.EndTime;
if (!segment.RequiresCleaving(out var totalPassCount, out var maximumPassCount))
{
continue;
}
if (subGrid.Cells.CleaveSegment(segment, NewSegmentsFromCleaving, PersistedClovenSegments, subGridSegmentPassCountLimit))
{
iterator.SegmentListExtended();
if (_segmentCleavingOperationsToLog)
{
_log.LogInformation(
$"Info: Performed cleave on segment ({cleavedTimeRangeStart}-{cleavedTimeRangeEnd}) of sub grid {ServerSubGridTree.GetLeafSubGridFullFileName(origin)}. TotalPassCount = {totalPassCount} MaximumPassCount = {maximumPassCount}");
}
}
else
{
// The segment cleave failed. While this is not a serious problem (as the sub grid will be
// cleaved at some point in the future when it is modified again via tag file processing etc)
// it will be noted in the log.
_log.LogWarning(
$"Cleave on segment ({cleavedTimeRangeStart}-{cleavedTimeRangeEnd}) of sub grid {ServerSubGridTree.GetLeafSubGridFullFileName(origin)} failed. TotalPassCount = {totalPassCount} MaximumPassCount = {maximumPassCount}");
}
if (_segmentCleavingOperationsToLog)
{
if (segment.RequiresCleaving(out totalPassCount, out maximumPassCount))
{
_log.LogWarning(
$"Cleave on segment ({cleavedTimeRangeStart}-{cleavedTimeRangeEnd}) of sub grid {subGrid.Moniker()} failed to reduce cell pass count below maximums (max passes = {totalPassCount}/{subGridSegmentPassCountLimit}, per cell = {maximumPassCount}/{_subGridMaxSegmentCellPassesLimit})");
}
}
} while (iterator.MoveToNextSubGridSegment());
}
/// <summary>
/// Orchestrates all the activities relating to saving the state of a created or modified sub grid, including
/// cleaving, saving updated elements, creating new elements and arranging for the retirement of
/// elements that have been replaced in the persistent store as a result of this activity.
/// </summary>
public bool SaveLeafSubGrid(IServerLeafSubGrid subGrid,
IStorageProxy storageProxyForSubGrids,
IStorageProxy storageProxyForSubGridSegments,
List <ISubGridSpatialAffinityKey> invalidatedSpatialStreams)
{
//Log.LogInformation($"Saving {subGrid.Moniker()} to persistent store");
try
{
// Perform segment cleaving as the first activity in the action of saving a leaf sub grid
// to disk. This reduces the number of segment cleaving actions that would otherwise
// be performed in the context of the aggregated integrator.
if (_segmentCleavingOperationsToLog)
{
_log.LogDebug($"About to perform segment cleaving on {subGrid.Moniker()}");
}
var cleaver = new SubGridSegmentCleaver();
cleaver.PerformSegmentCleaving(storageProxyForSubGridSegments, subGrid);
// Calculate the cell last pass information here, immediately before it is
// committed to the persistent store. The reason for this is to remove this
// compute intensive operation from the critical path in TAG file processing
// (which is the only writer of this information in the Raptor system).
// The computer is instructed to do a partial recompute, which will recompute
// all segments from the first segment marked as dirty.
subGrid.ComputeLatestPassInformation(false, storageProxyForSubGridSegments);
if (_segmentCleavingOperationsToLog)
{
_log.LogInformation($"SaveLeafSubGrid: {subGrid.Moniker()} ({subGrid.Cells.PassesData.Count} segments)");
}
var modifiedOriginalSegments = new List <ISubGridCellPassesDataSegment>(100);
var originAddress = new SubGridCellAddress(subGrid.OriginX, subGrid.OriginY);
// The following used to be an assert/exception. However, this is may readily
// happen if there are no modified segments resulting from processing a
// process TAG file where the Dirty flag for the sub grid is set but no cell
// passes are added to segments in that sub grid. As this is not an integrity
// issue the persistence of the modified sub grid is allowed, but noted in
// the log for posterity.
if (subGrid.Cells.PassesData.Count == 0)
{
_log.LogInformation(
$"Note: Saving a sub grid, {subGrid.Moniker()}, (Segments = {subGrid.Cells.PassesData.Count}, Dirty = {subGrid.Dirty}) with no cached sub grid segments to the persistent store in SaveLeafSubGrid (possible reprocessing of TAG file with no cell pass changes). " +
$"SubGrid.Directory.PersistedClovenSegments.Count={cleaver.PersistedClovenSegments?.Count}, ModifiedOriginalFiles.Count={modifiedOriginalSegments.Count}, NewSegmentsFromCleaving.Count={cleaver.NewSegmentsFromCleaving.Count}");
}
var iterator = new SubGridSegmentIterator(subGrid, storageProxyForSubGridSegments)
{
IterationDirection = IterationDirection.Forwards,
ReturnDirtyOnly = true,
};
//**********************************************************************
//***Construct list of original segment files that have been modified***
//*** These files may be updated in-situ with no sub grid/segment ***
//*** integrity issues wrt the segment directory in the sub grid ***
//**********************************************************************
iterator.MoveToFirstSubGridSegment();
while (iterator.CurrentSubGridSegment != null)
{
if (iterator.CurrentSubGridSegment.SegmentInfo.ExistsInPersistentStore && iterator.CurrentSubGridSegment.Dirty)
{
modifiedOriginalSegments.Add(iterator.CurrentSubGridSegment);
}
iterator.MoveToNextSubGridSegment();
}
//**********************************************************************
//*** Construct list of spatial streams that will be deleted or replaced
//*** in the FS file. These will be passed to the call that saves the
//*** sub grid directory file as an instruction to place them into the
//*** deferred deletion list
//**********************************************************************
lock (invalidatedSpatialStreams)
{
if (cleaver.PersistedClovenSegments != null)
{
invalidatedSpatialStreams.AddRange(cleaver.PersistedClovenSegments);
}
invalidatedSpatialStreams.AddRange(modifiedOriginalSegments.Select(x => x.SegmentInfo.AffinityKey(ID)));
}
if (cleaver.NewSegmentsFromCleaving.Count > 0)
{
//**********************************************************************
//*** Write new segment files generated by cleaving ***
//*** File system integrity failures here will have no effect on the ***
//*** sub grid/segment directory as they are not referenced by it. ***
//*** At worst they become orphans they may be cleaned by in the FS ***
//*** recovery phase ***
//**********************************************************************
if (_segmentCleavingOperationsToLog)
{
_log.LogInformation($"Sub grid has {cleaver.NewSegmentsFromCleaving.Count} new segments from cleaving");
}
foreach (var segment in cleaver.NewSegmentsFromCleaving)
{
// Update the version of the segment as it is about to be written
segment.SegmentInfo.Touch();
segment.SaveToFile(storageProxyForSubGridSegments, GetLeafSubGridSegmentFullFileName(originAddress, segment.SegmentInfo), out var fsError);
segment.Dirty = false;
if (fsError == FileSystemErrorStatus.OK)
{
if (_segmentCleavingOperationsToLog)
{
_log.LogInformation($"Saved new cloven grid segment file: {GetLeafSubGridSegmentFullFileName(originAddress, segment.SegmentInfo)}");
}
}
else
{
_log.LogWarning($"Failed to save cloven grid segment {GetLeafSubGridSegmentFullFileName(originAddress, segment.SegmentInfo)}: Error:{fsError}");
return(false);
}
}
}
if (modifiedOriginalSegments.Count > 0)
{
//**********************************************************************
//*** Write modified segment files ***
//*** File system integrity failures here will have no effect on the ***
//*** sub grid/segment directory as the previous version of the ***
//*** modified file being written will be recovered. ***
//**********************************************************************
if (_log.IsTraceEnabled())
{
_log.LogTrace($"Sub grid has {modifiedOriginalSegments.Count} modified segments");
}
foreach (var segment in modifiedOriginalSegments)
{
// Update the version of the segment as it is about to be written
segment.SegmentInfo.Touch();
segment.Dirty = false;
if (segment.SaveToFile(storageProxyForSubGridSegments, GetLeafSubGridSegmentFullFileName(originAddress, segment.SegmentInfo), out var fsError))
{
segment.Dirty = false;
if (_log.IsTraceEnabled())
{
_log.LogTrace($"Saved modified grid segment file: {segment}");
}
}
else
{
_log.LogError($"Failed to save modified original grid segment {GetLeafSubGridSegmentFullFileName(originAddress, segment.SegmentInfo)}: Error:{fsError}");
return(false);
}
}
}
// Any remaining dirty segments in the sub grid will be due to previously empty sub grids with newly
// created segments from ingest processing that have not required cleaving. These segments do not require any
// special treatment are jsut saved to persistent store
foreach (var segment in subGrid.Directory.SegmentDirectory.Select(x => x.Segment).Where(x => (x?.Dirty ?? false)))
{
if (segment.SaveToFile(storageProxyForSubGridSegments, GetLeafSubGridSegmentFullFileName(originAddress, segment.SegmentInfo), out var fsError))
{
segment.Dirty = false;
if (_log.IsTraceEnabled())
{
_log.LogTrace($"Saved new sub grid segment file: {segment}");
}
}
else
{
_log.LogError($"Failed to save new sub grid segment {GetLeafSubGridSegmentFullFileName(originAddress, segment.SegmentInfo)}: Error:{fsError}");
return(false);
}
}
//**********************************************************************
//*** Write the sub grid directory file ***
//**********************************************************************
/*
* There is no need to add the sub grid directory stream to the segment retirement
* queue as this will be automatically replaced when the new version of the
* sub grid directory is written to persistent store.
*
* // Add the stream representing the sub grid directory file to the list of
* // invalidated streams as this stream will be replaced with the stream
* // containing the updated directory information. Note: This only needs to
* // be done if the sub grid has previously been read from the FS file (if not
* // it has been created and not yet persisted to the store.
*
* if (subGrid.Directory.ExistsInPersistentStore)
* {
* // Include an additional invalidated spatial stream for the sub grid directory stream
* lock (invalidatedSpatialStreams)
* {
* invalidatedSpatialStreams.Add(subGrid.AffinityKey());
* }
* }
*/
if (subGrid.SaveDirectoryToFile(storageProxyForSubGrids, GetLeafSubGridFullFileName(originAddress)))
{
if (_log.IsTraceEnabled())
{
_log.LogTrace($"Saved grid directory file: {GetLeafSubGridFullFileName(originAddress)}");
}
}
else
{
if (_log.IsTraceEnabled())
{
_log.LogTrace($"Failed to save grid: {GetLeafSubGridFullFileName(originAddress)}");
}
return(false);
}
//**********************************************************************
//*** Reset segment dirty flags ***
//**********************************************************************
iterator.MoveToFirstSubGridSegment();
while (iterator.CurrentSubGridSegment != null)
{
iterator.CurrentSubGridSegment.Dirty = false;
iterator.CurrentSubGridSegment.SegmentInfo.ExistsInPersistentStore = true;
iterator.MoveToNextSubGridSegment();
}
//Log.LogInformation($"Completed saving {subGrid.Moniker()} to persistent store");
return(true);
}
catch (Exception e)
{
_log.LogError(e, "Exception raised in SaveLeafSubGrid");
}
return(false);
}
public bool LoadLeafSubGrid(IStorageProxy storageProxy,
SubGridCellAddress cellAddress,
bool loadAllPasses, bool loadLatestPasses,
IServerLeafSubGrid subGrid)
{
var fullFileName = string.Empty;
var result = false;
try
{
// Loading contents into a dirty sub grid (which should happen only on the mutable nodes), or
// when there is already content in the segment directory are strictly forbidden and break immutability
// rules for sub grids
if (subGrid.Dirty)
{
throw new TRexSubGridIOException("Leaf sub grid directory loads may not be performed while the sub grid is dirty. The information should be taken from the cache instead.");
}
// Load the cells into it from its file
// Briefly lock this sub grid just for the period required to read its contents
lock (subGrid)
{
// If more than thread wants this sub grid then they may concurrently attempt to load it.
// Make a check to see if this has happened and another thread has already loaded this sub grid directory.
// If so, just return success. Previously the commented out assert was enforced causing exceptions
// Debug.Assert(SubGrid.Directory?.SegmentDirectory?.Count == 0, "Loading a leaf sub grid directory when there are already segments present within it.");
// Check this thread is the winner of the lock to be able to load the contents
if (subGrid.Directory?.SegmentDirectory?.Count == 0)
{
// Ensure the appropriate storage is allocated
if (loadAllPasses)
{
subGrid.AllocateLeafFullPassStacks();
}
if (loadLatestPasses)
{
subGrid.AllocateLeafLatestPassGrid();
}
fullFileName = GetLeafSubGridFullFileName(cellAddress);
// Briefly lock this sub grid just for the period required to read its contents
result = subGrid.LoadDirectoryFromFile(storageProxy, fullFileName);
}
else
{
// A previous thread has already read the directory so there is nothing more to do, return a true result.
return(true);
}
}
}
finally
{
if (result && RecordSubGridFileReadingToLog)
{
_log.LogDebug($"Sub grid file {fullFileName} read from persistent store containing {subGrid.Directory.SegmentDirectory.Count} segments (Moniker: {subGrid.Moniker()}");
}
}
return(result);
}
public FileSystemErrorStatus LoadLeafSubGridSegment(IStorageProxy storageProxy,
SubGridCellAddress cellAddress,
bool loadLatestData,
bool loadAllPasses,
IServerLeafSubGrid subGrid,
ISubGridCellPassesDataSegment segment)
{
//Log.LogInformation($"Segment load on {cellAddress}:{Segment.SegmentInfo.StartTime}-{Segment.SegmentInfo.EndTime} beginning, loadLatestData = {loadLatestData}, loadAllPasses = {loadAllPasses}");
var needToLoadLatestData = loadLatestData && !segment.HasLatestData;
var needToLoadAllPasses = loadAllPasses && !segment.HasAllPasses;
if (!needToLoadLatestData && !needToLoadAllPasses)
{
//Log.LogInformation($"Segment load on {cellAddress} exiting as neither latest nor all passes required");
return(FileSystemErrorStatus.OK); // Nothing more to do here
}
// Lock the segment briefly while its contents is being loaded
lock (segment)
{
if (!(needToLoadLatestData ^ segment.HasLatestData) && !(needToLoadAllPasses ^ segment.HasAllPasses))
{
//Log.LogInformation($"Segment load on {cellAddress} leaving quietly as a previous thread has performed the load");
return(FileSystemErrorStatus.OK); // The load operation was performed on another thread. Leave quietly
}
// Ensure the appropriate storage is allocated
if (needToLoadLatestData)
{
segment.AllocateLatestPassGrid();
}
if (needToLoadAllPasses)
{
segment.AllocateFullPassStacks();
}
if (!segment.SegmentInfo.ExistsInPersistentStore)
{
//Log.LogInformation($"Segment load on {cellAddress} exiting as segment does not exist in persistent store");
return(FileSystemErrorStatus.OK); // Nothing more to do here
}
// Locate the segment file and load the data from it
var fullFileName = GetLeafSubGridSegmentFullFileName(cellAddress, segment.SegmentInfo);
// Load the cells into it from its file
var fsError = subGrid.LoadSegmentFromStorage(storageProxy, fullFileName, segment, needToLoadLatestData, needToLoadAllPasses);
if (fsError != FileSystemErrorStatus.OK)
{
//Log.LogInformation($"Segment load on {cellAddress} failed, performing allocation cleanup activities");
// Something bad happened. Remove the segment from the list. Return failure to the caller.
if (loadAllPasses)
{
segment.DeAllocateFullPassStacks();
}
if (loadLatestData)
{
segment.DeAllocateLatestPassGrid();
}
return(fsError);
}
}
//Log.LogInformation($"Segment load on {cellAddress} succeeded, AllPasses?={Segment.HasAllPasses}, Segment.PassesData?Null={Segment.PassesData==null} ");
return(FileSystemErrorStatus.OK);
}
/// <summary>
/// Orchestrates the mainline work of analyzing cell and cell pass information to create a client sub grid (heights, CMV, MDP etc)
/// based on filter and other information established in the class
/// </summary>
/// <param name="clientGrid"></param>
/// <param name="cellOverrideMask"></param>
/// <returns></returns>
public virtual ServerRequestResult RetrieveSubGrid(IClientLeafSubGrid clientGrid,
SubGridTreeBitmapSubGridBits cellOverrideMask,
out bool sieveFilterInUse,
Func <ServerRequestResult> computeSpatialFilterMaskAndClientProdDataMap)
{
sieveFilterInUse = false;
if (!Utilities.DerivedGridDataTypesAreCompatible(_gridDataType, clientGrid.GridDataType))
{
throw new TRexSubGridProcessingException($"Grid data type of client leaf sub grid [{clientGrid.GridDataType}] is not compatible with the grid data type of retriever [{_gridDataType}]");
}
var result = ServerRequestResult.UnknownError;
// SIGLogMessage.PublishNoODS(Nil, Format('In RetrieveSubGrid: Active pass filters = %s, Active cell filters = %s', [PassFilter.ActiveFiltersText, CellFilter.ActiveFiltersText]));
// Set up class local state for other methods to access
_clientGrid = clientGrid;
_clientGridAsLeaf = clientGrid as ClientLeafSubGrid;
_canUseGlobalLatestCells &=
!(_gridDataType == GridDataType.CCV ||
_gridDataType == GridDataType.CCVPercent) &&
_liftParams.CCVSummaryTypes != CCVSummaryTypes.None &&
!(_gridDataType == GridDataType.MDP ||
_gridDataType == GridDataType.MDPPercent) &&
_liftParams.MDPSummaryTypes != MDPSummaryTypes.None &&
!(_gridDataType == GridDataType.CCA || _gridDataType == GridDataType.CCAPercent) &&
!(_gridDataType == GridDataType.CellProfile ||
_gridDataType == GridDataType.PassCount ||
_gridDataType == GridDataType.CellPasses ||
_gridDataType == GridDataType.MachineSpeed ||
_gridDataType == GridDataType.CCVPercentChange ||
_gridDataType == GridDataType.MachineSpeedTarget ||
_gridDataType == GridDataType.CCVPercentChangeIgnoredTopNullValue);
// Support for lazy construction of any required profiling infrastructure
if (_clientGrid.WantsLiftProcessingResults() && _profiler == null)
{
// Some display types require lift processing to be able to select the
// appropriate cell pass containing the filtered value required.
_profiler = DIContext.Obtain <IProfilerBuilder <ProfileCell> >();
//TODO: should referenceDesignWrapper be null here or be passed through from args?
_profiler.Configure(ProfileStyle.CellPasses, _siteModel, _pdExistenceMap, _gridDataType, new FilterSet(_filter),
null, _populationControl, new CellPassFastEventLookerUpper(_siteModel), VolumeComputationType.None, _overrides, _liftParams);
_cellProfile = new ProfileCell();
// Create and configure the assignment context which is used to contain
// a filtered pass and its attendant machine events and target values
// prior to assignment to the client sub grid.
_assignmentContext.CellProfile = _cellProfile;
}
try
{
// Ensure pass type filter is set correctly
if (_filter.AttributeFilter.HasPassTypeFilter)
{
if ((_filter.AttributeFilter.PassTypeSet & (PassTypeSet.Front | PassTypeSet.Rear)) == PassTypeSet.Front)
{
_filter.AttributeFilter.PassTypeSet |= PassTypeSet.Rear; // these two types go together as half passes
}
}
// ... unless we can use the last pass grid to satisfy the query
if (_canUseGlobalLatestCells &&
!_filter.AttributeFilter.HasElevationRangeFilter &&
!_clientGrid.WantsLiftProcessingResults() &&
!_filter.AttributeFilter.HasElevationMappingModeFilter &&
!(_filter.AttributeFilter.HasElevationTypeFilter &&
(_filter.AttributeFilter.ElevationType == ElevationType.Highest ||
_filter.AttributeFilter.ElevationType == ElevationType.Lowest)) &&
!(_gridDataType == GridDataType.PassCount || _gridDataType == GridDataType.Temperature ||
_gridDataType == GridDataType.CellProfile || _gridDataType == GridDataType.CellPasses ||
_gridDataType == GridDataType.MachineSpeed))
{
_useLastPassGrid = true;
}
// First get the sub grid we are interested in
// SIGLogMessage.PublishNoODS(Nil, Format('Begin LocateSubGridContaining at %dx%d', [clientGrid.OriginX, clientGrid.OriginY]));
_subGrid = SubGridTrees.Server.Utilities.SubGridUtilities.LocateSubGridContaining(_storageProxy, _siteModel.Grid, clientGrid.OriginX, clientGrid.OriginY, _siteModel.Grid.NumLevels, false, false);
// SIGLogMessage.PublishNoODS(Nil, Format('End LocateSubGridContaining at %dx%d', [clientGrid.OriginX, clientGrid.Origin]));
if (_subGrid == null) // This should never really happen, but we'll be polite about it
{
Log.LogWarning(
$"Sub grid address (CellX={clientGrid.OriginX}, CellY={clientGrid.OriginY}) passed to LocateSubGridContaining() from RetrieveSubGrid() did not match an existing sub grid in the data model. Returning SubGridNotFound as response with a null sub grid reference.");
return(ServerRequestResult.SubGridNotFound);
}
// Now process the contents of that sub grid into the sub grid to be returned to the client.
if (!_subGrid.IsLeafSubGrid())
{
Log.LogInformation("Requests of node sub grids in the server sub grid are not yet supported");
return(result);
}
if (!(_subGrid is IServerLeafSubGrid))
{
Log.LogError($"_SubGrid {_subGrid.Moniker()} is not a server grid leaf node");
return(result);
}
// SIGLogMessage.PublishNoODS(Nil, Format('Getting sub grid leaf at %dx%d', [clientGrid.OriginX, clientGrid.OriginY]));
_subGridAsLeaf = (IServerLeafSubGrid)_subGrid;
_globalLatestCells = _subGridAsLeaf.Directory.GlobalLatestCells;
if (PruneSubGridRetrievalHere())
{
return(ServerRequestResult.NoError);
}
// SIGLogMessage.PublishNoODS(Nil, Format('Setup for stripe iteration at %dx%d', [clientGrid.OriginX, clientGrid.OriginY]));
SetupForCellPassStackExamination();
// Determine if a sieve filter is required for the sub grid where the sieve matches
// the X and Y pixel world size (used for WMS tile computation)
_subGrid.CalculateWorldOrigin(out var subGridWorldOriginX, out var subGridWorldOriginY);
sieveFilterInUse = _areaControlSet.UseIntegerAlgorithm
? GridRotationUtilities.ComputeSieveBitmaskInteger(subGridWorldOriginX, subGridWorldOriginY, _subGrid.Moniker(), _areaControlSet, _siteModel.CellSize, out _sieveBitmask)
: GridRotationUtilities.ComputeSieveBitmaskFloat(subGridWorldOriginX, subGridWorldOriginY, _areaControlSet, _siteModel.CellSize, _assignmentContext, out _sieveBitmask);
//if (Debug_ExtremeLogSwitchC) Log.LogDebug($"Performing stripe iteration at {clientGrid.OriginX}x{clientGrid.OriginY}");
if (computeSpatialFilterMaskAndClientProdDataMap != null)
{
_clientGrid.ProdDataMap.Assign(_subGridAsLeaf.Directory.GlobalLatestCells.PassDataExistenceMap);
var innerResult = computeSpatialFilterMaskAndClientProdDataMap();
if (innerResult != ServerRequestResult.NoError)
{
return(innerResult);
}
}
// Before iterating over stripes of this sub grid, compute a scan map detailing to the best of our current
// knowledge, which cells need to be visited so that only cells the filter wants and which are actually
// present in the data set are requested. If the intent is to store the result in a cache then ensure the
// entire content is requested for the sub grid.
if (_prepareGridForCacheStorageIfNoSieving)
{
_aggregatedCellScanMap.Fill();
}
else
{
_aggregatedCellScanMap.OrWith(_globalLatestCells.PassDataExistenceMap);
if (sieveFilterInUse)
{
_aggregatedCellScanMap.AndWith(_sieveBitmask); // ... and which are required by any sieve mask
}
_aggregatedCellScanMap.AndWith(_clientGridAsLeaf.ProdDataMap); // ... and which are in the required production data map
_aggregatedCellScanMap.AndWith(_clientGridAsLeaf.FilterMap); // ... and which are in the required filter map
}
// Iterate over the stripes in the sub grid processing each one in turn.
for (byte i = 0; i < SubGridTreeConsts.SubGridTreeDimension; i++)
{
RetrieveSubGridStripe(i);
}
//if Debug_ExtremeLogSwitchC then Log.LogDebug($"Stripe iteration complete at {clientGrid.OriginX}x{clientGrid.OriginY}");
result = ServerRequestResult.NoError;
}
catch (Exception e)
{
Log.LogError(e, $"Exception occurred in {nameof(RetrieveSubGrid)}");
throw;
}
return(result);
}
/// <summary>
/// Locates the sub grid in the sub grid tree that contains the cell identified by CellX and CellY in the global
/// sub grid tree cell address space. The tree level for the sub grid returned is specified in Level.
/// </summary>
public static ISubGrid LocateSubGridContaining(IStorageProxy storageProxyForSubGrids,
IServerSubGridTree forSubGridTree,
//const GridDataCache : TICDataStoreCache;
int cellX,
int cellY,
byte level,
bool lookInCacheOnly,
bool acceptSpeculativeReadFailure)
{
IServerLeafSubGrid leafSubGrid = null;
var createdANewSubGrid = false;
ISubGrid result = null;
if (forSubGridTree == null)
{
throw new TRexSubGridProcessingException($"Sub grid tree null in {nameof(LocateSubGridContaining)}");
}
// Note: Sub grid tree specific interlocks are no longer used. The tree now internally
// manages fine grained locks across structurally mutating activities such as node/leaf
// sub grid addition and reading content from the persistent store.
// First check to see if the requested cell is present in a leaf sub grid
var subGrid = forSubGridTree.LocateClosestSubGridContaining(cellX, cellY, level);
if (subGrid == null) // Something bad happened
{
_log.LogWarning($"Failed to locate sub grid at {cellX}:{cellY}, level {level}, data model ID:{forSubGridTree.ID}");
return(null);
}
if (!subGrid.IsLeafSubGrid() && !lookInCacheOnly && level == forSubGridTree.NumLevels)
{
if (forSubGridTree.CachingStrategy == ServerSubGridTreeCachingStrategy.CacheSubGridsInTree)
{
// Create the leaf sub grid that will be used to read in the sub grid from the disk.
// In the case where the sub grid isn't present on the disk this reference will be destroyed
subGrid = forSubGridTree.ConstructPathToCell(cellX, cellY, Types.SubGridPathConstructionType.CreateLeaf);
}
else if (forSubGridTree.CachingStrategy == ServerSubGridTreeCachingStrategy.CacheSubGridsInIgniteGridCache)
{
// Create the leaf sub grid without constructing elements in the grid to represent it other than the
// path in the tree to the parent of the sub grid.
// Note: Setting owner and parent relationship from the sub grid to the tree in this fashion permits
// business logic in th sub grid that require knowledge of it parent and owner relationships to function
// correctly while not including a reference to the sub grid from the tree.
subGrid = forSubGridTree.CreateNewSubGrid(forSubGridTree.NumLevels);
subGrid.SetAbsoluteOriginPosition(cellX & ~SubGridTreeConsts.SubGridLocalKeyMask, cellY & ~SubGridTreeConsts.SubGridLocalKeyMask);
subGrid.Owner = forSubGridTree;
subGrid.Parent = forSubGridTree.ConstructPathToCell(cellX, cellY, Types.SubGridPathConstructionType.CreatePathToLeaf);
}
if (subGrid != null)
{
createdANewSubGrid = true;
}
else
{
_log.LogError($"Failed to create leaf sub grid in LocateSubGridContaining for sub grid at {cellX}x{cellY}");
return(null);
}
}
if (subGrid.IsLeafSubGrid())
{
leafSubGrid = subGrid as IServerLeafSubGrid;
}
if (leafSubGrid == null) // Something bad happened
{
_log.LogError($"Sub grid request result for {cellX}:{cellY} is not a leaf sub grid, it is a {subGrid.GetType().Name}.");
return(null);
}
if (!createdANewSubGrid)
{
if (lookInCacheOnly)
{
if (subGrid.Level == level)
{
return(subGrid);
}
// If the returned sub grid is a leaf sub grid then it was already present in the
// cache. If the level of the returned sub grid matches the request level parameter
// then there is nothing more to do here.
if (subGrid.IsLeafSubGrid() &&
((leafSubGrid.HasSubGridDirectoryDetails || leafSubGrid.Dirty) &&
leafSubGrid.HasAllCellPasses() && leafSubGrid.HasLatestData()) ||
(!subGrid.IsLeafSubGrid() && subGrid.Level == level))
{
return(subGrid);
}
}
}
if ((!leafSubGrid.HasSubGridDirectoryDetails && !leafSubGrid.Dirty) ||
!(leafSubGrid.HasAllCellPasses() && leafSubGrid.HasLatestData()))
{
// The requested cell is either not present in the sub grid tree (cache),
// or it is residing on disk, and a newly created sub grid has been constructed
// to contain the data read from disk.
// The underlying assumption is that this method is only called if the caller knows
// that the sub grid exists in the sub grid tree (this is known via the sub grid existence
// map available to the caller). In cases where eventual consistency
// may mean that a sub grid was removed from the sub grid tree since the caller retrieved
// its copy of the sub grid existence map this function will fail gracefully with a null sub grid.
// The exception to this rule is the tag file processor service which may speculatively
// attempt to read a sub grid that doesn't exist.
// This is a different approach to desktop systems where the individual node sub grids
// contain mini existence maps for the sub grids below them.
if (forSubGridTree.LoadLeafSubGrid(storageProxyForSubGrids,
new SubGridCellAddress(cellX, cellY),
true, true,
leafSubGrid))
{
// We've loaded it - get the reference to the new sub grid and return it
result = leafSubGrid;
}
else
{
// The sub grid could not be loaded. This is likely due to it not ever existing
// in the model, or it may have been deleted. Failure here does not necessarily
// constitute evidence of corruption in the data model. Examination of the
// spatial existence map in conjunction with the requested sub grid index is
// required to determine that. Advise the caller nothing was read by sending back
// a null sub grid reference.
// The failed sub grid is not proactively deleted and will remain so the normal cache
// expiry mechanism can remove it in its normal operations
if (acceptSpeculativeReadFailure)
{
// Return the otherwise empty sub grid back to the caller and integrate it into the cache
if (_log.IsTraceEnabled())
{
_log.LogTrace($"Speculative read failure accepted for sub grid {leafSubGrid.Moniker()}. Blank sub grid returned to caller.");
}
result = leafSubGrid;
}
else
{
_log.LogWarning($"Failed to read leaf sub grid {leafSubGrid.Moniker()} in model {forSubGridTree.ID}. Failed sub grid is NOT removed from the tree");
// Empty the sub grid leaf based data to encourage it to be read on a secondary attempt
leafSubGrid.DeAllocateLeafFullPassStacks();
leafSubGrid.DeAllocateLeafLatestPassGrid();
}
}
}
// Ignite special case - allow Dirty leaf sub grids to be returned
if (result == null)
{
if (leafSubGrid.HasSubGridDirectoryDetails && leafSubGrid.Dirty && leafSubGrid.HasAllCellPasses() && leafSubGrid.HasLatestData())
{
result = leafSubGrid;
}
}
// IGNITE: Last gasp - if the sub grid is in memory and has directory details then just return it
if (result == null && leafSubGrid.HasSubGridDirectoryDetails)
{
result = leafSubGrid;
}
return(result);
}
/// <summary>
/// Builds a fully analyzed vector of profiled cells from the list of cell passed to it
/// </summary>
public override bool Analyze(List <SummaryVolumeProfileCell> profileCells, ISubGridSegmentCellPassIterator cellPassIterator)
{
Log.LogDebug($"Analyze Summary Volume ProfileCells. Processing {profileCells.Count}");
var CurrentSubgridOrigin = new SubGridCellAddress(int.MaxValue, int.MaxValue);
ISubGrid SubGrid = null;
IServerLeafSubGrid _SubGridAsLeaf = null;
profileCell = null;
// Construct the set of requestors to query elevation sub grids needed for the summary volume calculations.
var filterSet = FilterUtilities.ConstructFilters(FilterSet, VolumeType);
IntermediaryFilterRequired = filterSet.Filters.Length == 3;
var utilities = DIContext.Obtain <IRequestorUtilities>();
Requestors = utilities.ConstructRequestors(null, SiteModel, Overrides, LiftParams,
utilities.ConstructRequestorIntermediaries(SiteModel, filterSet, true, GridDataType.HeightAndTime),
AreaControlSet.CreateAreaControlSet(), PDExistenceMap);
var cellOverrideMask = new SubGridTreeBitmapSubGridBits(SubGridBitsCreationOptions.Unfilled);
for (int I = 0; I < profileCells.Count; I++)
{
profileCell = profileCells[I];
// get sub grid origin for cell address
var thisSubgridOrigin = new SubGridCellAddress(profileCell.OTGCellX >> SubGridTreeConsts.SubGridIndexBitsPerLevel,
profileCell.OTGCellY >> SubGridTreeConsts.SubGridIndexBitsPerLevel);
if (!CurrentSubgridOrigin.Equals(thisSubgridOrigin)) // if we have a new sub grid to fetch
{
// if we have an existing sub grid and a change in sub grid detected process the current sub grid profile cell list
if (SubGrid != null)
{
ProcessSubGroup(new SubGridCellAddress(CurrentSubgridOrigin.X << SubGridTreeConsts.SubGridIndexBitsPerLevel, CurrentSubgridOrigin.Y << SubGridTreeConsts.SubGridIndexBitsPerLevel),
PDExistenceMap[CurrentSubgridOrigin.X, CurrentSubgridOrigin.Y], cellOverrideMask);
cellOverrideMask.Clear();
}
SubGrid = null;
cellCounter = 0;
// Does the sub grid tree contain this node in it's existence map? if so get sub grid
if (PDExistenceMap[thisSubgridOrigin.X, thisSubgridOrigin.Y])
{
SubGrid = SubGridTrees.Server.Utilities.SubGridUtilities.LocateSubGridContaining
(SiteModel.PrimaryStorageProxy, SiteModel.Grid, profileCell.OTGCellX, profileCell.OTGCellY, SiteModel.Grid.NumLevels, false, false);
}
_SubGridAsLeaf = SubGrid as ServerSubGridTreeLeaf;
if (_SubGridAsLeaf == null)
{
continue;
}
CurrentSubgridOrigin = thisSubgridOrigin; // all good to proceed with this sub grid
}
profileCellList[cellCounter++] = profileCell; // add cell to list to process for this sub grid
cellOverrideMask.SetBit(profileCell.OTGCellX & SubGridTreeConsts.SubGridLocalKeyMask, profileCell.OTGCellY & SubGridTreeConsts.SubGridLocalKeyMask);
}
if (cellCounter > 0 && SubGrid != null) // Make sure we process last list
{
ProcessSubGroup(new SubGridCellAddress(CurrentSubgridOrigin.X << SubGridTreeConsts.SubGridIndexBitsPerLevel, CurrentSubgridOrigin.Y << SubGridTreeConsts.SubGridIndexBitsPerLevel),
PDExistenceMap[CurrentSubgridOrigin.X, CurrentSubgridOrigin.Y], cellOverrideMask);
}
return(true);
}
/// <summary>
/// Set up a model with a single sub grid with a single cell containing 3 cell passes
/// </summary>
/// <returns></returns>
private ISiteModel CreateSiteModelWithSingleCellForTesting()
{
var siteModel = DIContext.Obtain <ISiteModels>().GetSiteModel(DITagFileFixture.NewSiteModelGuid, true);
// Switch to mutable storage representation to allow creation of content in the site model
siteModel.StorageRepresentationToSupply.Should().Be(StorageMutability.Immutable);
siteModel.SetStorageRepresentationToSupply(StorageMutability.Mutable);
siteModel.Machines.CreateNew("Test Machine", "", MachineType.Dozer, DeviceTypeEnum.SNM940, false, Guid.NewGuid());
// vibrationState is needed to get cmv values
siteModel.MachinesTargetValues[0].VibrationStateEvents.PutValueAtDate(BASE_TIME, VibrationState.On);
siteModel.MachinesTargetValues[0].AutoVibrationStateEvents.PutValueAtDate(BASE_TIME, AutoVibrationState.Manual);
// Construct the sub grid to hold the cell being tested
IServerLeafSubGrid leaf = siteModel.Grid.ConstructPathToCell(SubGridTreeConsts.DefaultIndexOriginOffset, SubGridTreeConsts.DefaultIndexOriginOffset, SubGridPathConstructionType.CreateLeaf) as IServerLeafSubGrid;
leaf.Should().NotBeNull();
leaf.AllocateLeafFullPassStacks();
leaf.CreateDefaultSegment();
leaf.AllocateFullPassStacks(leaf.Directory.SegmentDirectory.First());
leaf.AllocateLeafLatestPassGrid();
// Add the leaf to the site model existence map
siteModel.ExistenceMap[leaf.OriginX >> SubGridTreeConsts.SubGridIndexBitsPerLevel, leaf.OriginY >> SubGridTreeConsts.SubGridIndexBitsPerLevel] = true;
siteModel.Grid.CountLeafSubGridsInMemory().Should().Be(1);
// Add three passes, each separated by 10 seconds and descending by 100mm each pass
for (int i = 0; i < PASSES_IN_DECREMENTING_ELEVATION_LIST; i++)
{
leaf.AddPass(0, 0, new CellPass
{
InternalSiteModelMachineIndex = 0,
Time = BASE_TIME.AddSeconds(i * TIME_INCREMENT_SECONDS),
Height = BASE_HEIGHT + i * HEIGHT_DECREMENT,
PassType = PassType.Front,
CCV = CCV_Test,
MDP = MDP_Test,
MaterialTemperature = Temperature_Test
});
}
var cellPasses = leaf.Cells.PassesData[0].PassesData.ExtractCellPasses(0, 0);
cellPasses.Passes.Count.Should().Be(PASSES_IN_DECREMENTING_ELEVATION_LIST);
// Assign global latest cell pass to the appropriate pass
leaf.Directory.GlobalLatestCells[0, 0] = cellPasses.Passes.Last();
// Ensure the pass data existence map records the existence of a non null value in the cell
leaf.Directory.GlobalLatestCells.PassDataExistenceMap[0, 0] = true;
// Count the number of non-null elevation cells to verify a correct setup
long totalCells = 0;
siteModel.Grid.Root.ScanSubGrids(siteModel.Grid.FullCellExtent(), x => {
totalCells += leaf.Directory.GlobalLatestCells.PassDataExistenceMap.CountBits();
return(true);
});
totalCells.Should().Be(1);
return(siteModel);
}
