/// <summary>
/// Cleaves all dirty segments requiring cleaving within the given sub grid
/// </summary>
public void PerformSegmentCleaving(IStorageProxy storageProxyForSubGridSegments, IServerLeafSubGrid subGrid, int subGridSegmentPassCountLimit = 0)
{
var iterator = new SubGridSegmentIterator(subGrid, storageProxyForSubGridSegments)
{
IterationDirection = IterationDirection.Forwards,
ReturnDirtyOnly = true,
RetrieveAllPasses = true
};
var origin = new SubGridCellAddress(subGrid.OriginX, subGrid.OriginY);
if (!iterator.MoveToFirstSubGridSegment())
{
return;
}
do
{
var segment = iterator.CurrentSubGridSegment;
var cleavedTimeRangeStart = segment.SegmentInfo.StartTime;
var cleavedTimeRangeEnd = segment.SegmentInfo.EndTime;
if (!segment.RequiresCleaving(out var totalPassCount, out var maximumPassCount))
{
continue;
}
if (subGrid.Cells.CleaveSegment(segment, NewSegmentsFromCleaving, PersistedClovenSegments, subGridSegmentPassCountLimit))
{
iterator.SegmentListExtended();
if (_segmentCleavingOperationsToLog)
{
_log.LogInformation(
$"Info: Performed cleave on segment ({cleavedTimeRangeStart}-{cleavedTimeRangeEnd}) of sub grid {ServerSubGridTree.GetLeafSubGridFullFileName(origin)}. TotalPassCount = {totalPassCount} MaximumPassCount = {maximumPassCount}");
}
}
else
{
// The segment cleave failed. While this is not a serious problem (as the sub grid will be
// cleaved at some point in the future when it is modified again via tag file processing etc)
// it will be noted in the log.
_log.LogWarning(
$"Cleave on segment ({cleavedTimeRangeStart}-{cleavedTimeRangeEnd}) of sub grid {ServerSubGridTree.GetLeafSubGridFullFileName(origin)} failed. TotalPassCount = {totalPassCount} MaximumPassCount = {maximumPassCount}");
}
if (_segmentCleavingOperationsToLog)
{
if (segment.RequiresCleaving(out totalPassCount, out maximumPassCount))
{
_log.LogWarning(
$"Cleave on segment ({cleavedTimeRangeStart}-{cleavedTimeRangeEnd}) of sub grid {subGrid.Moniker()} failed to reduce cell pass count below maximums (max passes = {totalPassCount}/{subGridSegmentPassCountLimit}, per cell = {maximumPassCount}/{_subGridMaxSegmentCellPassesLimit})");
}
}
} while (iterator.MoveToNextSubGridSegment());
}
/// <summary>
/// Orchestrates all the activities relating to saving the state of a created or modified sub grid, including
/// cleaving, saving updated elements, creating new elements and arranging for the retirement of
/// elements that have been replaced in the persistent store as a result of this activity.
/// </summary>
public bool SaveLeafSubGrid(IServerLeafSubGrid subGrid,
IStorageProxy storageProxyForSubGrids,
IStorageProxy storageProxyForSubGridSegments,
List <ISubGridSpatialAffinityKey> invalidatedSpatialStreams)
{
//Log.LogInformation($"Saving {subGrid.Moniker()} to persistent store");
try
{
// Perform segment cleaving as the first activity in the action of saving a leaf sub grid
// to disk. This reduces the number of segment cleaving actions that would otherwise
// be performed in the context of the aggregated integrator.
if (_segmentCleavingOperationsToLog)
{
_log.LogDebug($"About to perform segment cleaving on {subGrid.Moniker()}");
}
var cleaver = new SubGridSegmentCleaver();
cleaver.PerformSegmentCleaving(storageProxyForSubGridSegments, subGrid);
// Calculate the cell last pass information here, immediately before it is
// committed to the persistent store. The reason for this is to remove this
// compute intensive operation from the critical path in TAG file processing
// (which is the only writer of this information in the Raptor system).
// The computer is instructed to do a partial recompute, which will recompute
// all segments from the first segment marked as dirty.
subGrid.ComputeLatestPassInformation(false, storageProxyForSubGridSegments);
if (_segmentCleavingOperationsToLog)
{
_log.LogInformation($"SaveLeafSubGrid: {subGrid.Moniker()} ({subGrid.Cells.PassesData.Count} segments)");
}
var modifiedOriginalSegments = new List <ISubGridCellPassesDataSegment>(100);
var originAddress = new SubGridCellAddress(subGrid.OriginX, subGrid.OriginY);
// The following used to be an assert/exception. However, this is may readily
// happen if there are no modified segments resulting from processing a
// process TAG file where the Dirty flag for the sub grid is set but no cell
// passes are added to segments in that sub grid. As this is not an integrity
// issue the persistence of the modified sub grid is allowed, but noted in
// the log for posterity.
if (subGrid.Cells.PassesData.Count == 0)
{
_log.LogInformation(
$"Note: Saving a sub grid, {subGrid.Moniker()}, (Segments = {subGrid.Cells.PassesData.Count}, Dirty = {subGrid.Dirty}) with no cached sub grid segments to the persistent store in SaveLeafSubGrid (possible reprocessing of TAG file with no cell pass changes). " +
$"SubGrid.Directory.PersistedClovenSegments.Count={cleaver.PersistedClovenSegments?.Count}, ModifiedOriginalFiles.Count={modifiedOriginalSegments.Count}, NewSegmentsFromCleaving.Count={cleaver.NewSegmentsFromCleaving.Count}");
}
var iterator = new SubGridSegmentIterator(subGrid, storageProxyForSubGridSegments)
{
IterationDirection = IterationDirection.Forwards,
ReturnDirtyOnly = true,
};
//**********************************************************************
//***Construct list of original segment files that have been modified***
//*** These files may be updated in-situ with no sub grid/segment ***
//*** integrity issues wrt the segment directory in the sub grid ***
//**********************************************************************
iterator.MoveToFirstSubGridSegment();
while (iterator.CurrentSubGridSegment != null)
{
if (iterator.CurrentSubGridSegment.SegmentInfo.ExistsInPersistentStore && iterator.CurrentSubGridSegment.Dirty)
{
modifiedOriginalSegments.Add(iterator.CurrentSubGridSegment);
}
iterator.MoveToNextSubGridSegment();
}
//**********************************************************************
//*** Construct list of spatial streams that will be deleted or replaced
//*** in the FS file. These will be passed to the call that saves the
//*** sub grid directory file as an instruction to place them into the
//*** deferred deletion list
//**********************************************************************
lock (invalidatedSpatialStreams)
{
if (cleaver.PersistedClovenSegments != null)
{
invalidatedSpatialStreams.AddRange(cleaver.PersistedClovenSegments);
}
invalidatedSpatialStreams.AddRange(modifiedOriginalSegments.Select(x => x.SegmentInfo.AffinityKey(ID)));
}
if (cleaver.NewSegmentsFromCleaving.Count > 0)
{
//**********************************************************************
//*** Write new segment files generated by cleaving ***
//*** File system integrity failures here will have no effect on the ***
//*** sub grid/segment directory as they are not referenced by it. ***
//*** At worst they become orphans they may be cleaned by in the FS ***
//*** recovery phase ***
//**********************************************************************
if (_segmentCleavingOperationsToLog)
{
_log.LogInformation($"Sub grid has {cleaver.NewSegmentsFromCleaving.Count} new segments from cleaving");
}
foreach (var segment in cleaver.NewSegmentsFromCleaving)
{
// Update the version of the segment as it is about to be written
segment.SegmentInfo.Touch();
segment.SaveToFile(storageProxyForSubGridSegments, GetLeafSubGridSegmentFullFileName(originAddress, segment.SegmentInfo), out var fsError);
segment.Dirty = false;
if (fsError == FileSystemErrorStatus.OK)
{
if (_segmentCleavingOperationsToLog)
{
_log.LogInformation($"Saved new cloven grid segment file: {GetLeafSubGridSegmentFullFileName(originAddress, segment.SegmentInfo)}");
}
}
else
{
_log.LogWarning($"Failed to save cloven grid segment {GetLeafSubGridSegmentFullFileName(originAddress, segment.SegmentInfo)}: Error:{fsError}");
return(false);
}
}
}
if (modifiedOriginalSegments.Count > 0)
{
//**********************************************************************
//*** Write modified segment files ***
//*** File system integrity failures here will have no effect on the ***
//*** sub grid/segment directory as the previous version of the ***
//*** modified file being written will be recovered. ***
//**********************************************************************
if (_log.IsTraceEnabled())
{
_log.LogTrace($"Sub grid has {modifiedOriginalSegments.Count} modified segments");
}
foreach (var segment in modifiedOriginalSegments)
{
// Update the version of the segment as it is about to be written
segment.SegmentInfo.Touch();
segment.Dirty = false;
if (segment.SaveToFile(storageProxyForSubGridSegments, GetLeafSubGridSegmentFullFileName(originAddress, segment.SegmentInfo), out var fsError))
{
segment.Dirty = false;
if (_log.IsTraceEnabled())
{
_log.LogTrace($"Saved modified grid segment file: {segment}");
}
}
else
{
_log.LogError($"Failed to save modified original grid segment {GetLeafSubGridSegmentFullFileName(originAddress, segment.SegmentInfo)}: Error:{fsError}");
return(false);
}
}
}
// Any remaining dirty segments in the sub grid will be due to previously empty sub grids with newly
// created segments from ingest processing that have not required cleaving. These segments do not require any
// special treatment are jsut saved to persistent store
foreach (var segment in subGrid.Directory.SegmentDirectory.Select(x => x.Segment).Where(x => (x?.Dirty ?? false)))
{
if (segment.SaveToFile(storageProxyForSubGridSegments, GetLeafSubGridSegmentFullFileName(originAddress, segment.SegmentInfo), out var fsError))
{
segment.Dirty = false;
if (_log.IsTraceEnabled())
{
_log.LogTrace($"Saved new sub grid segment file: {segment}");
}
}
else
{
_log.LogError($"Failed to save new sub grid segment {GetLeafSubGridSegmentFullFileName(originAddress, segment.SegmentInfo)}: Error:{fsError}");
return(false);
}
}
//**********************************************************************
//*** Write the sub grid directory file ***
//**********************************************************************
/*
* There is no need to add the sub grid directory stream to the segment retirement
* queue as this will be automatically replaced when the new version of the
* sub grid directory is written to persistent store.
*
* // Add the stream representing the sub grid directory file to the list of
* // invalidated streams as this stream will be replaced with the stream
* // containing the updated directory information. Note: This only needs to
* // be done if the sub grid has previously been read from the FS file (if not
* // it has been created and not yet persisted to the store.
*
* if (subGrid.Directory.ExistsInPersistentStore)
* {
* // Include an additional invalidated spatial stream for the sub grid directory stream
* lock (invalidatedSpatialStreams)
* {
* invalidatedSpatialStreams.Add(subGrid.AffinityKey());
* }
* }
*/
if (subGrid.SaveDirectoryToFile(storageProxyForSubGrids, GetLeafSubGridFullFileName(originAddress)))
{
if (_log.IsTraceEnabled())
{
_log.LogTrace($"Saved grid directory file: {GetLeafSubGridFullFileName(originAddress)}");
}
}
else
{
if (_log.IsTraceEnabled())
{
_log.LogTrace($"Failed to save grid: {GetLeafSubGridFullFileName(originAddress)}");
}
return(false);
}
//**********************************************************************
//*** Reset segment dirty flags ***
//**********************************************************************
iterator.MoveToFirstSubGridSegment();
while (iterator.CurrentSubGridSegment != null)
{
iterator.CurrentSubGridSegment.Dirty = false;
iterator.CurrentSubGridSegment.SegmentInfo.ExistsInPersistentStore = true;
iterator.MoveToNextSubGridSegment();
}
//Log.LogInformation($"Completed saving {subGrid.Moniker()} to persistent store");
return(true);
}
catch (Exception e)
{
_log.LogError(e, "Exception raised in SaveLeafSubGrid");
}
return(false);
}
/// <summary>
/// Locates the sub grid in the sub grid tree that contains the cell identified by CellX and CellY in the global
/// sub grid tree cell address space. The tree level for the sub grid returned is specified in Level.
/// </summary>
public static ISubGrid LocateSubGridContaining(IStorageProxy storageProxyForSubGrids,
IServerSubGridTree forSubGridTree,
//const GridDataCache : TICDataStoreCache;
int cellX,
int cellY,
byte level,
bool lookInCacheOnly,
bool acceptSpeculativeReadFailure)
{
IServerLeafSubGrid leafSubGrid = null;
var createdANewSubGrid = false;
ISubGrid result = null;
if (forSubGridTree == null)
{
throw new TRexSubGridProcessingException($"Sub grid tree null in {nameof(LocateSubGridContaining)}");
}
// Note: Sub grid tree specific interlocks are no longer used. The tree now internally
// manages fine grained locks across structurally mutating activities such as node/leaf
// sub grid addition and reading content from the persistent store.
// First check to see if the requested cell is present in a leaf sub grid
var subGrid = forSubGridTree.LocateClosestSubGridContaining(cellX, cellY, level);
if (subGrid == null) // Something bad happened
{
_log.LogWarning($"Failed to locate sub grid at {cellX}:{cellY}, level {level}, data model ID:{forSubGridTree.ID}");
return(null);
}
if (!subGrid.IsLeafSubGrid() && !lookInCacheOnly && level == forSubGridTree.NumLevels)
{
if (forSubGridTree.CachingStrategy == ServerSubGridTreeCachingStrategy.CacheSubGridsInTree)
{
// Create the leaf sub grid that will be used to read in the sub grid from the disk.
// In the case where the sub grid isn't present on the disk this reference will be destroyed
subGrid = forSubGridTree.ConstructPathToCell(cellX, cellY, Types.SubGridPathConstructionType.CreateLeaf);
}
else if (forSubGridTree.CachingStrategy == ServerSubGridTreeCachingStrategy.CacheSubGridsInIgniteGridCache)
{
// Create the leaf sub grid without constructing elements in the grid to represent it other than the
// path in the tree to the parent of the sub grid.
// Note: Setting owner and parent relationship from the sub grid to the tree in this fashion permits
// business logic in th sub grid that require knowledge of it parent and owner relationships to function
// correctly while not including a reference to the sub grid from the tree.
subGrid = forSubGridTree.CreateNewSubGrid(forSubGridTree.NumLevels);
subGrid.SetAbsoluteOriginPosition(cellX & ~SubGridTreeConsts.SubGridLocalKeyMask, cellY & ~SubGridTreeConsts.SubGridLocalKeyMask);
subGrid.Owner = forSubGridTree;
subGrid.Parent = forSubGridTree.ConstructPathToCell(cellX, cellY, Types.SubGridPathConstructionType.CreatePathToLeaf);
}
if (subGrid != null)
{
createdANewSubGrid = true;
}
else
{
_log.LogError($"Failed to create leaf sub grid in LocateSubGridContaining for sub grid at {cellX}x{cellY}");
return(null);
}
}
if (subGrid.IsLeafSubGrid())
{
leafSubGrid = subGrid as IServerLeafSubGrid;
}
if (leafSubGrid == null) // Something bad happened
{
_log.LogError($"Sub grid request result for {cellX}:{cellY} is not a leaf sub grid, it is a {subGrid.GetType().Name}.");
return(null);
}
if (!createdANewSubGrid)
{
if (lookInCacheOnly)
{
if (subGrid.Level == level)
{
return(subGrid);
}
// If the returned sub grid is a leaf sub grid then it was already present in the
// cache. If the level of the returned sub grid matches the request level parameter
// then there is nothing more to do here.
if (subGrid.IsLeafSubGrid() &&
((leafSubGrid.HasSubGridDirectoryDetails || leafSubGrid.Dirty) &&
leafSubGrid.HasAllCellPasses() && leafSubGrid.HasLatestData()) ||
(!subGrid.IsLeafSubGrid() && subGrid.Level == level))
{
return(subGrid);
}
}
}
if ((!leafSubGrid.HasSubGridDirectoryDetails && !leafSubGrid.Dirty) ||
!(leafSubGrid.HasAllCellPasses() && leafSubGrid.HasLatestData()))
{
// The requested cell is either not present in the sub grid tree (cache),
// or it is residing on disk, and a newly created sub grid has been constructed
// to contain the data read from disk.
// The underlying assumption is that this method is only called if the caller knows
// that the sub grid exists in the sub grid tree (this is known via the sub grid existence
// map available to the caller). In cases where eventual consistency
// may mean that a sub grid was removed from the sub grid tree since the caller retrieved
// its copy of the sub grid existence map this function will fail gracefully with a null sub grid.
// The exception to this rule is the tag file processor service which may speculatively
// attempt to read a sub grid that doesn't exist.
// This is a different approach to desktop systems where the individual node sub grids
// contain mini existence maps for the sub grids below them.
if (forSubGridTree.LoadLeafSubGrid(storageProxyForSubGrids,
new SubGridCellAddress(cellX, cellY),
true, true,
leafSubGrid))
{
// We've loaded it - get the reference to the new sub grid and return it
result = leafSubGrid;
}
else
{
// The sub grid could not be loaded. This is likely due to it not ever existing
// in the model, or it may have been deleted. Failure here does not necessarily
// constitute evidence of corruption in the data model. Examination of the
// spatial existence map in conjunction with the requested sub grid index is
// required to determine that. Advise the caller nothing was read by sending back
// a null sub grid reference.
// The failed sub grid is not proactively deleted and will remain so the normal cache
// expiry mechanism can remove it in its normal operations
if (acceptSpeculativeReadFailure)
{
// Return the otherwise empty sub grid back to the caller and integrate it into the cache
if (_log.IsTraceEnabled())
{
_log.LogTrace($"Speculative read failure accepted for sub grid {leafSubGrid.Moniker()}. Blank sub grid returned to caller.");
}
result = leafSubGrid;
}
else
{
_log.LogWarning($"Failed to read leaf sub grid {leafSubGrid.Moniker()} in model {forSubGridTree.ID}. Failed sub grid is NOT removed from the tree");
// Empty the sub grid leaf based data to encourage it to be read on a secondary attempt
leafSubGrid.DeAllocateLeafFullPassStacks();
leafSubGrid.DeAllocateLeafLatestPassGrid();
}
}
}
// Ignite special case - allow Dirty leaf sub grids to be returned
if (result == null)
{
if (leafSubGrid.HasSubGridDirectoryDetails && leafSubGrid.Dirty && leafSubGrid.HasAllCellPasses() && leafSubGrid.HasLatestData())
{
result = leafSubGrid;
}
}
// IGNITE: Last gasp - if the sub grid is in memory and has directory details then just return it
if (result == null && leafSubGrid.HasSubGridDirectoryDetails)
{
result = leafSubGrid;
}
return(result);
}
public bool LoadLeafSubGrid(IStorageProxy storageProxy,
SubGridCellAddress cellAddress,
bool loadAllPasses, bool loadLatestPasses,
IServerLeafSubGrid subGrid)
{
var fullFileName = string.Empty;
var result = false;
try
{
// Loading contents into a dirty sub grid (which should happen only on the mutable nodes), or
// when there is already content in the segment directory are strictly forbidden and break immutability
// rules for sub grids
if (subGrid.Dirty)
{
throw new TRexSubGridIOException("Leaf sub grid directory loads may not be performed while the sub grid is dirty. The information should be taken from the cache instead.");
}
// Load the cells into it from its file
// Briefly lock this sub grid just for the period required to read its contents
lock (subGrid)
{
// If more than thread wants this sub grid then they may concurrently attempt to load it.
// Make a check to see if this has happened and another thread has already loaded this sub grid directory.
// If so, just return success. Previously the commented out assert was enforced causing exceptions
// Debug.Assert(SubGrid.Directory?.SegmentDirectory?.Count == 0, "Loading a leaf sub grid directory when there are already segments present within it.");
// Check this thread is the winner of the lock to be able to load the contents
if (subGrid.Directory?.SegmentDirectory?.Count == 0)
{
// Ensure the appropriate storage is allocated
if (loadAllPasses)
{
subGrid.AllocateLeafFullPassStacks();
}
if (loadLatestPasses)
{
subGrid.AllocateLeafLatestPassGrid();
}
fullFileName = GetLeafSubGridFullFileName(cellAddress);
// Briefly lock this sub grid just for the period required to read its contents
result = subGrid.LoadDirectoryFromFile(storageProxy, fullFileName);
}
else
{
// A previous thread has already read the directory so there is nothing more to do, return a true result.
return(true);
}
}
}
finally
{
if (result && RecordSubGridFileReadingToLog)
{
_log.LogDebug($"Sub grid file {fullFileName} read from persistent store containing {subGrid.Directory.SegmentDirectory.Count} segments (Moniker: {subGrid.Moniker()}");
}
}
return(result);
}
public ISubGridCellPassesDataSegment AddNewSegment(IServerLeafSubGrid subGrid,
ISubGridCellPassesDataSegmentInfo segmentInfo)
{
if (segmentInfo == null)
{
throw new TRexSubGridProcessingException($"Null segment info passed to AddNewSegment for sub grid {subGrid.Moniker()}");
}
if (segmentInfo.Segment != null)
{
throw new TRexSubGridProcessingException($"Segment info passed to AddNewSegment for sub grid {subGrid.Moniker()} already contains an allocated segment");
}
var Result = new SubGridCellPassesDataSegment
{
Owner = subGrid,
SegmentInfo = segmentInfo
};
segmentInfo.Segment = Result;
for (int I = 0; I < Count; I++)
{
if (segmentInfo.EndTime <= Items[I].SegmentInfo.StartTime)
{
Items.Insert(I, Result);
if (_performSegmentAdditionIntegrityChecks)
{
for (int J = 0; J < Count - 1; J++)
{
if (Items[J].SegmentInfo.StartTime >= Items[J + 1].SegmentInfo.StartTime)
{
_log.LogError($"Segment passes list out of order {Items[J].SegmentInfo.StartTime} versus {Items[J + 1].SegmentInfo.StartTime}. Segment count = {Count}");
DumpSegmentsToLog();
throw new TRexSubGridProcessingException($"Segment passes list out of order {Items[J].SegmentInfo.StartTime} versus {Items[J + 1].SegmentInfo.StartTime}. Segment count = {Count}");
}
}
}
return(Result);
}
}
// if we get to here, then the new segment is at the end of the list, so just add it to the end
Add(Result);
return(Result);
}