public void Test_SubgridSegment_Cleaver()
{
// Create a sub grid to hold the segment
IServerLeafSubGrid subGrid = MakeSubgridWith10240CellPassesAtOneSecondIntervals();
// Exercise the cleaver!
// Instruct the segment container to cleave the segment
// Set the segment to not dirty - it should be ignored
subGrid.Cells.PassesData[0].Dirty = false;
var cleaver = new SubGridSegmentCleaver();
cleaver.PerformSegmentCleaving(StorageProxy.Instance(StorageMutability.Mutable), subGrid, 10000);
Assert.True(1 == subGrid.Cells.PassesData.Count, $"After cleaving with no dirty segments there are {subGrid.Cells.PassesData.Count} segments instead of the expected one segments");
// Set the segment to not dirty - it should be ignored
subGrid.Cells.PassesData[0].Dirty = true;
cleaver = new SubGridSegmentCleaver();
cleaver.PerformSegmentCleaving(StorageProxy.Instance(StorageMutability.Mutable), subGrid, 10000);
//Check there are now two segments in total
Assert.True(2 == subGrid.Cells.PassesData.Count, $"After cleaving there are {subGrid.Cells.PassesData.Count} segments instead of the expected two segments");
//Check the total number of passes across the two segments is 10240, and the maximum pass count is 5
ISubGridCellPassesDataSegment segment1 = subGrid.Cells.PassesData[0];
ISubGridCellPassesDataSegment segment2 = subGrid.Cells.PassesData[1];
segment1.PassesData.CalculateTotalPasses(out int totalPassCount1, out _, out int maximumPassCount1);
segment2.PassesData.CalculateTotalPasses(out int totalPassCount2, out _, out int maximumPassCount2);
Assert.True(10240 == (totalPassCount1 + totalPassCount2), $"Totals ({totalPassCount1} and {totalPassCount2} don't add up to 10240 after cleaving");
Assert.True(5 == maximumPassCount1, $"Maximum pass count 1 {maximumPassCount1}, is not 5");
Assert.True(5 == maximumPassCount2, $"Maximum pass count 2 {maximumPassCount2}, is not 5");
// Check the segment pass count in the segment is correct
Assert.True(totalPassCount1 == segment1.PassesData.SegmentPassCount, $"Total passes for segment 1 {totalPassCount1} is not equal to segmentPassCount in that segment {segment1.PassesData.SegmentPassCount}");
Assert.True(totalPassCount2 == segment2.PassesData.SegmentPassCount, $"Total passes for segment 2 {totalPassCount2} is not equal to segmentPassCount in that segment {segment2.PassesData.SegmentPassCount}");
}
/// <summary>
/// Orchestrates all the activities relating to saving the state of a created or modified sub grid, including
/// cleaving, saving updated elements, creating new elements and arranging for the retirement of
/// elements that have been replaced in the persistent store as a result of this activity.
/// </summary>
public bool SaveLeafSubGrid(IServerLeafSubGrid subGrid,
IStorageProxy storageProxyForSubGrids,
IStorageProxy storageProxyForSubGridSegments,
List <ISubGridSpatialAffinityKey> invalidatedSpatialStreams)
{
//Log.LogInformation($"Saving {subGrid.Moniker()} to persistent store");
try
{
// Perform segment cleaving as the first activity in the action of saving a leaf sub grid
// to disk. This reduces the number of segment cleaving actions that would otherwise
// be performed in the context of the aggregated integrator.
if (_segmentCleavingOperationsToLog)
{
_log.LogDebug($"About to perform segment cleaving on {subGrid.Moniker()}");
}
var cleaver = new SubGridSegmentCleaver();
cleaver.PerformSegmentCleaving(storageProxyForSubGridSegments, subGrid);
// Calculate the cell last pass information here, immediately before it is
// committed to the persistent store. The reason for this is to remove this
// compute intensive operation from the critical path in TAG file processing
// (which is the only writer of this information in the Raptor system).
// The computer is instructed to do a partial recompute, which will recompute
// all segments from the first segment marked as dirty.
subGrid.ComputeLatestPassInformation(false, storageProxyForSubGridSegments);
if (_segmentCleavingOperationsToLog)
{
_log.LogInformation($"SaveLeafSubGrid: {subGrid.Moniker()} ({subGrid.Cells.PassesData.Count} segments)");
}
var modifiedOriginalSegments = new List <ISubGridCellPassesDataSegment>(100);
var originAddress = new SubGridCellAddress(subGrid.OriginX, subGrid.OriginY);
// The following used to be an assert/exception. However, this is may readily
// happen if there are no modified segments resulting from processing a
// process TAG file where the Dirty flag for the sub grid is set but no cell
// passes are added to segments in that sub grid. As this is not an integrity
// issue the persistence of the modified sub grid is allowed, but noted in
// the log for posterity.
if (subGrid.Cells.PassesData.Count == 0)
{
_log.LogInformation(
$"Note: Saving a sub grid, {subGrid.Moniker()}, (Segments = {subGrid.Cells.PassesData.Count}, Dirty = {subGrid.Dirty}) with no cached sub grid segments to the persistent store in SaveLeafSubGrid (possible reprocessing of TAG file with no cell pass changes). " +
$"SubGrid.Directory.PersistedClovenSegments.Count={cleaver.PersistedClovenSegments?.Count}, ModifiedOriginalFiles.Count={modifiedOriginalSegments.Count}, NewSegmentsFromCleaving.Count={cleaver.NewSegmentsFromCleaving.Count}");
}
var iterator = new SubGridSegmentIterator(subGrid, storageProxyForSubGridSegments)
{
IterationDirection = IterationDirection.Forwards,
ReturnDirtyOnly = true,
};
//**********************************************************************
//***Construct list of original segment files that have been modified***
//*** These files may be updated in-situ with no sub grid/segment ***
//*** integrity issues wrt the segment directory in the sub grid ***
//**********************************************************************
iterator.MoveToFirstSubGridSegment();
while (iterator.CurrentSubGridSegment != null)
{
if (iterator.CurrentSubGridSegment.SegmentInfo.ExistsInPersistentStore && iterator.CurrentSubGridSegment.Dirty)
{
modifiedOriginalSegments.Add(iterator.CurrentSubGridSegment);
}
iterator.MoveToNextSubGridSegment();
}
//**********************************************************************
//*** Construct list of spatial streams that will be deleted or replaced
//*** in the FS file. These will be passed to the call that saves the
//*** sub grid directory file as an instruction to place them into the
//*** deferred deletion list
//**********************************************************************
lock (invalidatedSpatialStreams)
{
if (cleaver.PersistedClovenSegments != null)
{
invalidatedSpatialStreams.AddRange(cleaver.PersistedClovenSegments);
}
invalidatedSpatialStreams.AddRange(modifiedOriginalSegments.Select(x => x.SegmentInfo.AffinityKey(ID)));
}
if (cleaver.NewSegmentsFromCleaving.Count > 0)
{
//**********************************************************************
//*** Write new segment files generated by cleaving ***
//*** File system integrity failures here will have no effect on the ***
//*** sub grid/segment directory as they are not referenced by it. ***
//*** At worst they become orphans they may be cleaned by in the FS ***
//*** recovery phase ***
//**********************************************************************
if (_segmentCleavingOperationsToLog)
{
_log.LogInformation($"Sub grid has {cleaver.NewSegmentsFromCleaving.Count} new segments from cleaving");
}
foreach (var segment in cleaver.NewSegmentsFromCleaving)
{
// Update the version of the segment as it is about to be written
segment.SegmentInfo.Touch();
segment.SaveToFile(storageProxyForSubGridSegments, GetLeafSubGridSegmentFullFileName(originAddress, segment.SegmentInfo), out var fsError);
segment.Dirty = false;
if (fsError == FileSystemErrorStatus.OK)
{
if (_segmentCleavingOperationsToLog)
{
_log.LogInformation($"Saved new cloven grid segment file: {GetLeafSubGridSegmentFullFileName(originAddress, segment.SegmentInfo)}");
}
}
else
{
_log.LogWarning($"Failed to save cloven grid segment {GetLeafSubGridSegmentFullFileName(originAddress, segment.SegmentInfo)}: Error:{fsError}");
return(false);
}
}
}
if (modifiedOriginalSegments.Count > 0)
{
//**********************************************************************
//*** Write modified segment files ***
//*** File system integrity failures here will have no effect on the ***
//*** sub grid/segment directory as the previous version of the ***
//*** modified file being written will be recovered. ***
//**********************************************************************
if (_log.IsTraceEnabled())
{
_log.LogTrace($"Sub grid has {modifiedOriginalSegments.Count} modified segments");
}
foreach (var segment in modifiedOriginalSegments)
{
// Update the version of the segment as it is about to be written
segment.SegmentInfo.Touch();
segment.Dirty = false;
if (segment.SaveToFile(storageProxyForSubGridSegments, GetLeafSubGridSegmentFullFileName(originAddress, segment.SegmentInfo), out var fsError))
{
segment.Dirty = false;
if (_log.IsTraceEnabled())
{
_log.LogTrace($"Saved modified grid segment file: {segment}");
}
}
else
{
_log.LogError($"Failed to save modified original grid segment {GetLeafSubGridSegmentFullFileName(originAddress, segment.SegmentInfo)}: Error:{fsError}");
return(false);
}
}
}
// Any remaining dirty segments in the sub grid will be due to previously empty sub grids with newly
// created segments from ingest processing that have not required cleaving. These segments do not require any
// special treatment are jsut saved to persistent store
foreach (var segment in subGrid.Directory.SegmentDirectory.Select(x => x.Segment).Where(x => (x?.Dirty ?? false)))
{
if (segment.SaveToFile(storageProxyForSubGridSegments, GetLeafSubGridSegmentFullFileName(originAddress, segment.SegmentInfo), out var fsError))
{
segment.Dirty = false;
if (_log.IsTraceEnabled())
{
_log.LogTrace($"Saved new sub grid segment file: {segment}");
}
}
else
{
_log.LogError($"Failed to save new sub grid segment {GetLeafSubGridSegmentFullFileName(originAddress, segment.SegmentInfo)}: Error:{fsError}");
return(false);
}
}
//**********************************************************************
//*** Write the sub grid directory file ***
//**********************************************************************
/*
* There is no need to add the sub grid directory stream to the segment retirement
* queue as this will be automatically replaced when the new version of the
* sub grid directory is written to persistent store.
*
* // Add the stream representing the sub grid directory file to the list of
* // invalidated streams as this stream will be replaced with the stream
* // containing the updated directory information. Note: This only needs to
* // be done if the sub grid has previously been read from the FS file (if not
* // it has been created and not yet persisted to the store.
*
* if (subGrid.Directory.ExistsInPersistentStore)
* {
* // Include an additional invalidated spatial stream for the sub grid directory stream
* lock (invalidatedSpatialStreams)
* {
* invalidatedSpatialStreams.Add(subGrid.AffinityKey());
* }
* }
*/
if (subGrid.SaveDirectoryToFile(storageProxyForSubGrids, GetLeafSubGridFullFileName(originAddress)))
{
if (_log.IsTraceEnabled())
{
_log.LogTrace($"Saved grid directory file: {GetLeafSubGridFullFileName(originAddress)}");
}
}
else
{
if (_log.IsTraceEnabled())
{
_log.LogTrace($"Failed to save grid: {GetLeafSubGridFullFileName(originAddress)}");
}
return(false);
}
//**********************************************************************
//*** Reset segment dirty flags ***
//**********************************************************************
iterator.MoveToFirstSubGridSegment();
while (iterator.CurrentSubGridSegment != null)
{
iterator.CurrentSubGridSegment.Dirty = false;
iterator.CurrentSubGridSegment.SegmentInfo.ExistsInPersistentStore = true;
iterator.MoveToNextSubGridSegment();
}
//Log.LogInformation($"Completed saving {subGrid.Moniker()} to persistent store");
return(true);
}
catch (Exception e)
{
_log.LogError(e, "Exception raised in SaveLeafSubGrid");
}
return(false);
}