/// <summary>
/// Create and configure the segment iterator to be used
/// </summary>
/// <param name="passFilter"></param>
private void SetupForCellPassStackExamination(ICellPassAttributeFilter passFilter)
{
SegmentIterator = new SubGridSegmentIterator(null, null, SiteModel.PrimaryStorageProxy);
if (passFilter.ReturnEarliestFilteredCellPass ||
(passFilter.HasElevationTypeFilter && passFilter.ElevationType == ElevationType.First))
{
SegmentIterator.IterationDirection = IterationDirection.Forwards;
}
else
{
SegmentIterator.IterationDirection = IterationDirection.Backwards;
}
if (passFilter.HasMachineFilter)
{
if (passFilter.SiteModel == null)
{
passFilter.SiteModel = SiteModel;
}
SegmentIterator.SetMachineRestriction(passFilter.GetMachineIDsSet());
}
// Create and configure the cell pass iterator to be used
CellPassIterator = new SubGridSegmentCellPassIterator_NonStatic
{
SegmentIterator = SegmentIterator
};
CellPassIterator.SetTimeRange(passFilter.HasTimeFilter, passFilter.StartTime, passFilter.EndTime);
}
protected override void SetupForCellPassStackExamination()
{
base.SetupForCellPassStackExamination();
if (!_commonCellPassStackExaminationDone)
{
// Modify the cell pass iterator to obey the time range established by the StartDate/EndDate arguments
_cellPassIterator.SetTimeRange(true, StartDate, EndDate);
// Construct a reversing segment and cell pass iterator used to locate necessary cell passes earlier than the first cell pass
// according to the primary cell pass iterator _cellPassIterator. Allow it's time range to be all of history prior to StartDate
// Note that both the reversing and progressive sub grid operator are provide the same sub grid. This permits both iterators
// to leverage the inherent cache of segment information within the sub grid.
_reversingSegmentIterator = new SubGridSegmentIterator(_subGridAsLeaf, _subGridAsLeaf.Directory, _storageProxy)
{
IterationDirection = IterationDirection.Backwards
};
_reversingCellPassIterator = new SubGridSegmentCellPassIterator_NonStatic(_reversingSegmentIterator, _maxNumberOfPassesToReturn);
_reversingCellPassIterator.SetTimeRange(true, DateTime.MinValue, StartDate.AddTicks(-100));
_commonCellPassStackExaminationDone = true;
}
_segmentIterator.IterationDirection = IterationDirection.Forwards;
_reversingSegmentIterator.SubGrid = _subGridAsLeaf;
_reversingSegmentIterator.Directory = _subGridAsLeaf.Directory;
}
private void IntegrateIntoLiveGrid(IServerLeafSubGrid sourceSubGrid,
ISubGridSegmentIterator segmentIterator,
Action <int, int> subGridChangeNotifier)
{
var targetSubGrid = LocateOrCreateSubGrid(_target, sourceSubGrid.OriginX, sourceSubGrid.OriginY);
try
{
if (targetSubGrid == null)
{
_log.LogError("Failed to locate or create sub grid in IntegrateIntoLiveGrid");
return;
}
if (!IntegrateIntoLiveDatabase(sourceSubGrid, targetSubGrid, segmentIterator, subGridChangeNotifier))
{
_log.LogError("Integration into live database failed");
}
}
finally
{
// At this point TargetSubGrid is of no further interest. The site model will be dropped in-toto once all
// changes have been integrated into the live sub grids and releasing it's resources here alleviates memory
// pressure that occurs if all sub grids are integrated before any resource freeing occurs
// This is not mediated by a using block as we want the sub grid to remain in the sub grid tree (ie:
// removing it is messy and not necessary here, it can be delegated to the DropSiteModel() phase.
targetSubGrid?.DeAllocateLeafFullPassStacks();
targetSubGrid?.DeAllocateLeafLatestPassGrid();
}
}
protected virtual void SetupForCellPassStackExamination()
{
if (!_commonCellPassStackExaminationDone)
{
_populationControl.PreparePopulationControl(_gridDataType, _liftParams, _filter.AttributeFilter, _clientGrid.EventPopulationFlags);
_filter.AttributeFilter.RequestedGridDataType = _gridDataType;
// Create and configure the segment iterator to be used
_segmentIterator = new SubGridSegmentIterator(_subGridAsLeaf, _subGridAsLeaf.Directory, _storageProxy);
if (_filter.AttributeFilter.HasMachineFilter)
{
_segmentIterator.SetMachineRestriction(_filter.AttributeFilter.GetMachineIDsSet());
}
// Create and configure the cell pass iterator to be used
_cellPassIterator = new SubGridSegmentCellPassIterator_NonStatic(_segmentIterator, _maxNumberOfPassesToReturn);
_cellPassIterator.SetTimeRange(_filter.AttributeFilter.HasTimeFilter, _filter.AttributeFilter.StartTime, _filter.AttributeFilter.EndTime);
_commonCellPassStackExaminationDone = true;
}
if (_filter.AttributeFilter.ReturnEarliestFilteredCellPass ||
(_filter.AttributeFilter.HasElevationTypeFilter && _filter.AttributeFilter.ElevationType == ElevationType.First))
{
_segmentIterator.IterationDirection = IterationDirection.Forwards;
}
else
{
_segmentIterator.IterationDirection = IterationDirection.Backwards;
}
_segmentIterator.SubGrid = _subGridAsLeaf;
_segmentIterator.Directory = _subGridAsLeaf.Directory;
// Some display types require lift processing to be able to select the appropriate cell pass containing the filtered value required.
if (_clientGrid.WantsLiftProcessingResults())
{
_segmentIterator.IterationDirection = IterationDirection.Forwards;
_cellPassIterator.MaxNumberOfPassesToReturn = _maxNumberOfPassesToReturn;
}
}
protected SubGridRetrieverBase(ISiteModel siteModel,
GridDataType gridDataType,
ICombinedFilter filter,
ICellPassAttributeFilterProcessingAnnex filterAnnex,
bool hasOverrideSpatialCellRestriction,
BoundingIntegerExtent2D overrideSpatialCellRestriction,
bool prepareGridForCacheStorageIfNoSieving,
int maxNumberOfPassesToReturn,
IStorageProxy storageProxy,
AreaControlSet areaControlSet,
IFilteredValuePopulationControl populationControl,
ISubGridTreeBitMask pDExistenceMap,
IOverrideParameters overrides,
ILiftParameters liftParams)
{
_segmentIterator = null;
_cellPassIterator = null;
_siteModel = siteModel;
_gridDataType = gridDataType;
_filter = filter;
_filterAnnex = filterAnnex;
_hasOverrideSpatialCellRestriction = hasOverrideSpatialCellRestriction;
_overrideSpatialCellRestriction = overrideSpatialCellRestriction;
_prepareGridForCacheStorageIfNoSieving = prepareGridForCacheStorageIfNoSieving;
_maxNumberOfPassesToReturn = maxNumberOfPassesToReturn;
_storageProxy = storageProxy;
_populationControl = populationControl;
_areaControlSet = areaControlSet;
_pdExistenceMap = pDExistenceMap;
_overrides = overrides;
_liftParams = liftParams;
// Create and configure the assignment context which is used to contain a filtered pass and
// its attendant machine events and target values prior to assignment to the client sub grid.
_assignmentContext = new FilteredValueAssignmentContext {
Overrides = overrides, LiftParams = liftParams
};
_filter.AttributeFilter.SiteModel = siteModel;
_canUseGlobalLatestCells = _filter.AttributeFilter.LastRecordedCellPassSatisfiesFilter;
}
private bool IntegrateIntoLiveDatabase(IServerLeafSubGrid sourceSubGrid,
IServerLeafSubGrid targetSubGrid,
ISubGridSegmentIterator segmentIterator,
Action <int, int> subGridChangeNotifier)
{
// Note the fact that this sub grid will be changed and become dirty as a result
// of the cell pass integration
targetSubGrid.SetDirty();
targetSubGrid.Integrate(sourceSubGrid, segmentIterator, false);
subGridChangeNotifier?.Invoke(targetSubGrid.OriginX, targetSubGrid.OriginY);
// Save the integrated state of the sub grid segments to allow Ignite to store & socialize the update
// within the cluster.
// Failure to save a piece of data aborts the entire integration
var result = false;
if (_target.SaveLeafSubGrid(targetSubGrid, _storageProxySubGrids, _storageProxySubGridSegments, InvalidatedSpatialStreams))
{
// Successfully saving the sub grid directory information is the point at which this sub grid may be recognized to exist
// in the site model. Note this by including it within the SiteModel existence map
_siteModel.ExistenceMap.SetCell(targetSubGrid.OriginX >> SubGridTreeConsts.SubGridIndexBitsPerLevel,
targetSubGrid.OriginY >> SubGridTreeConsts.SubGridIndexBitsPerLevel,
true);
result = true;
}
else
{
_log.LogError($"Sub grid leaf save failed for {targetSubGrid}, existence map not modified.");
}
// Finally, mark the source sub grid as not being dirty. We need to do this to allow
// the sub grid to permit its destruction as all changes have been merged into the target.
if (result)
{
sourceSubGrid.AllChangesMigrated();
}
return(result);
}
private void IntegrateSubGrid(IServerLeafSubGrid sourceSubGrid,
SubGridTreeIntegrationMode integrationMode,
Action <int, int> subGridChangeNotifier,
ISubGridSegmentIterator segmentIterator)
{
// Locate a matching sub grid in this tree. If there is none, then create it
// and assign the sub grid from the iterator to it. If there is one, process
// the cell pass stacks merging the two together
var integratingIntoIntermediaryGrid = integrationMode == SubGridTreeIntegrationMode.UsingInMemoryTarget;
if (integratingIntoIntermediaryGrid)
{
IntegrateIntoIntermediaryGrid(sourceSubGrid, segmentIterator);
}
else
{
IntegrateIntoLiveGrid(sourceSubGrid, segmentIterator, subGridChangeNotifier);
}
}
private void IntegrateIntoIntermediaryGrid(IServerLeafSubGrid sourceSubGrid, ISubGridSegmentIterator segmentIterator)
{
var targetSubGrid = _target.ConstructPathToCell(sourceSubGrid.OriginX, sourceSubGrid.OriginY,
SubGridPathConstructionType.CreateLeaf) as IServerLeafSubGrid;
targetSubGrid.AllocateLeafFullPassStacks();
// If the node is brand new (ie: it does not have any cell passes committed to it yet)
// then create and select the default segment
if (targetSubGrid.Directory.SegmentDirectory.Count == 0)
{
targetSubGrid.Cells.SelectSegment(Consts.MIN_DATETIME_AS_UTC);
targetSubGrid.Cells.PassesData[0].AllocateFullPassStacks();
}
if (targetSubGrid.Cells.PassesData[0].PassesData == null)
{
_log.LogCritical("No segment passes data in new segment");
return;
}
targetSubGrid.Integrate(sourceSubGrid, segmentIterator, true);
}
/// <summary>
/// Construct a cell pass iterator using a given segment iterator and an optional maximum number of passes to return
/// in the course of the iteration
/// </summary>
/// <param name="iterator"></param>
/// <param name="maxNumberOfPassesToReturn"></param>
public SubGridSegmentCellPassIterator_NonStatic(ISubGridSegmentIterator iterator, int maxNumberOfPassesToReturn = int.MaxValue) : base(iterator, maxNumberOfPassesToReturn)
{
}
/// <summary>
/// Construct a cell pass iterator using a given segment iterator and an optional maximum number of passes to return
/// in the course of the iteration
/// </summary>
/// <param name="iterator"></param>
/// <param name="maxNumberOfPassesToReturn"></param>
protected SubGridSegmentCellPassIterator_Base(ISubGridSegmentIterator iterator,
int maxNumberOfPassesToReturn = int.MaxValue) : this()
{
SegmentIterator = iterator;
MaxNumberOfPassesToReturn = maxNumberOfPassesToReturn;
}
