public SubGridCellPassesDataSegmentInfo(DateTime startTime, DateTime endTime,
ISubGridCellPassesDataSegment segment) : this()
{
StartTime = startTime;
EndTime = endTime;
Segment = segment;
}
public void MetaTest_Test_MakeSubgridWith10240CellPassesAtOneSecondIntervals()
{
// Create a subgrid to hold the segment
IServerLeafSubGrid subGrid = MakeSubgridWith10240CellPassesAtOneSecondIntervals();
ISubGridCellPassesDataSegment segment = subGrid.Cells.PassesData[0];
// Check all cells have exactly 10 passes
VSS.TRex.SubGridTrees.Core.Utilities.SubGridUtilities.SubGridDimensionalIterator((x, y) =>
{
Assert.True(segment.PassesData.PassCount(x, y) == 10,
$"Cell in segment at {x},{y} does not have 10 cell passes");
});
//Check the total number of passes is 10240, and the maximum pass count is 10
segment.PassesData.CalculateTotalPasses(out int totalPassCount, out _, out int maximumPassCount);
Assert.True(10240 == totalPassCount, "Initial total pass count not 10240");
Assert.True(10240 == segment.PassesData.SegmentPassCount,
$"segment.PassesData.SegmentPassCount does not equal 10240 (it is {segment.PassesData.SegmentPassCount})");
Assert.True(10 == maximumPassCount, "Initial maximum pass count not 10");
// Check the time range is as expected
segment.PassesData.CalculateTimeRange(out DateTime startSegmentTime, out DateTime endSegmentTime);
Assert.True(endSegmentTime > startSegmentTime, $"End time {endSegmentTime} not greater than startTime {startSegmentTime}");
Assert.True(startSegmentTime == startTime, $"Start time {startSegmentTime} not equal to {startTime} as expected");
Assert.True(endSegmentTime == startTime.AddSeconds(10239), $"End time {endSegmentTime} not equal to {startTime.AddSeconds(10239)} as expected");
}
public int Add(ISubGridCellPassesDataSegment item)
{
int Index = Count - 1;
while (Index >= 0 && item.SegmentInfo.StartTime < Items[Index].SegmentInfo.StartTime)
{
Index--;
}
Index++;
Items.Insert(Index, item);
return(Index);
/*
* {$IFDEF DEBUG}
* Counter := 0;
* for Index := 0 to Count - 2 do
* if Items[Index].SegmentInfo.StartTime >= Items[Index + 1].SegmentInfo.StartTime then
* begin
* SIGLogMessage.PublishNoODS(Self, Format('Segment passes list out of order %.6f versus %.6f. Segment count = %d', { SKIP}
* [Items[Index].SegmentInfo.StartTime, Items[Index + 1].SegmentInfo.StartTime, Count]), slmcAssert);
* Inc(Counter);
* end;
* if Counter > 0 then
* DumpSegmentsToLog;
* {$ENDIF}
*/
}
public void Test_SubgridSegment_Cleaver()
{
// Create a sub grid to hold the segment
IServerLeafSubGrid subGrid = MakeSubgridWith10240CellPassesAtOneSecondIntervals();
// Exercise the cleaver!
// Instruct the segment container to cleave the segment
// Set the segment to not dirty - it should be ignored
subGrid.Cells.PassesData[0].Dirty = false;
var cleaver = new SubGridSegmentCleaver();
cleaver.PerformSegmentCleaving(StorageProxy.Instance(StorageMutability.Mutable), subGrid, 10000);
Assert.True(1 == subGrid.Cells.PassesData.Count, $"After cleaving with no dirty segments there are {subGrid.Cells.PassesData.Count} segments instead of the expected one segments");
// Set the segment to not dirty - it should be ignored
subGrid.Cells.PassesData[0].Dirty = true;
cleaver = new SubGridSegmentCleaver();
cleaver.PerformSegmentCleaving(StorageProxy.Instance(StorageMutability.Mutable), subGrid, 10000);
//Check there are now two segments in total
Assert.True(2 == subGrid.Cells.PassesData.Count, $"After cleaving there are {subGrid.Cells.PassesData.Count} segments instead of the expected two segments");
//Check the total number of passes across the two segments is 10240, and the maximum pass count is 5
ISubGridCellPassesDataSegment segment1 = subGrid.Cells.PassesData[0];
ISubGridCellPassesDataSegment segment2 = subGrid.Cells.PassesData[1];
segment1.PassesData.CalculateTotalPasses(out int totalPassCount1, out _, out int maximumPassCount1);
segment2.PassesData.CalculateTotalPasses(out int totalPassCount2, out _, out int maximumPassCount2);
Assert.True(10240 == (totalPassCount1 + totalPassCount2), $"Totals ({totalPassCount1} and {totalPassCount2} don't add up to 10240 after cleaving");
Assert.True(5 == maximumPassCount1, $"Maximum pass count 1 {maximumPassCount1}, is not 5");
Assert.True(5 == maximumPassCount2, $"Maximum pass count 2 {maximumPassCount2}, is not 5");
// Check the segment pass count in the segment is correct
Assert.True(totalPassCount1 == segment1.PassesData.SegmentPassCount, $"Total passes for segment 1 {totalPassCount1} is not equal to segmentPassCount in that segment {segment1.PassesData.SegmentPassCount}");
Assert.True(totalPassCount2 == segment2.PassesData.SegmentPassCount, $"Total passes for segment 2 {totalPassCount2} is not equal to segmentPassCount in that segment {segment2.PassesData.SegmentPassCount}");
}
public void Test_SubgridSegmentCleaving()
{
// Create a sub grid to hold the segment. the subGrid retains the limit in the subGrid constructor in MakeSubgridWith10240CellPassesAtOneSecondIntervals()
IServerLeafSubGrid subGrid = MakeSubgridWith10240CellPassesAtOneSecondIntervals();
ISubGridCellPassesDataSegment segment = subGrid.Cells.PassesData[0];
// Instruct the segment container to cleave the segment with a limit of 100000
var newSegmentsFromCleaving = new List <ISubGridCellPassesDataSegment>();
var persistedClovenSegments = new List <ISubGridSpatialAffinityKey>();
Assert.False(subGrid.Cells.CleaveSegment(segment, newSegmentsFromCleaving, persistedClovenSegments, 100000), "Segment was cloven when cell pass count was below limit");
// Set the cleaving limit to 10000 to force the segment TO BE cloven = the cleave result should be true
subGrid = MakeSubgridWith10240CellPassesAtOneSecondIntervals();
segment = subGrid.Cells.PassesData[0];
newSegmentsFromCleaving = new List <ISubGridCellPassesDataSegment>();
persistedClovenSegments = new List <ISubGridSpatialAffinityKey>();
Assert.True(subGrid.Cells.CleaveSegment(segment, newSegmentsFromCleaving, persistedClovenSegments, 10000), "Segment failed to cleave with pass count above limit");
//Check there are now two segments in total
Assert.True(2 == subGrid.Cells.PassesData.Count, $"After cleaving there are {subGrid.Cells.PassesData.Count} segments instead of the expected two segments");
//Check the total number of passes across the two segments is 10240, and the maximum pass count is 5
ISubGridCellPassesDataSegment segment1 = subGrid.Cells.PassesData[0];
ISubGridCellPassesDataSegment segment2 = subGrid.Cells.PassesData[1];
segment1.PassesData.CalculateTotalPasses(out int totalPassCount1, out _, out int maximumPassCount1);
segment2.PassesData.CalculateTotalPasses(out int totalPassCount2, out _, out int maximumPassCount2);
Assert.True(10240 == totalPassCount1 + totalPassCount2, $"Totals ({totalPassCount1} and {totalPassCount2} don't add up to 10240 after cleaving");
Assert.True(5 == maximumPassCount1, $"Maximum pass count 1 {maximumPassCount1}, is not 5");
Assert.True(5 == maximumPassCount2, $"Maximum pass count 2 {maximumPassCount2}, is not 5");
// Check the segment pass count in the segment is correct
Assert.True(totalPassCount1 == segment1.PassesData.SegmentPassCount, $"Total passes for segment 1 {totalPassCount1} is not equal to segmentPassCount in that segment {segment1.PassesData.SegmentPassCount}");
Assert.True(totalPassCount2 == segment2.PassesData.SegmentPassCount, $"Total passes for segment 2 {totalPassCount2} is not equal to segmentPassCount in that segment {segment2.PassesData.SegmentPassCount}");
}
public FileSystemErrorStatus LoadLeafSubGridSegment(IStorageProxy storageProxy,
SubGridCellAddress cellAddress,
bool loadLatestData,
bool loadAllPasses,
IServerLeafSubGrid subGrid,
ISubGridCellPassesDataSegment segment)
{
//Log.LogInformation($"Segment load on {cellAddress}:{Segment.SegmentInfo.StartTime}-{Segment.SegmentInfo.EndTime} beginning, loadLatestData = {loadLatestData}, loadAllPasses = {loadAllPasses}");
var needToLoadLatestData = loadLatestData && !segment.HasLatestData;
var needToLoadAllPasses = loadAllPasses && !segment.HasAllPasses;
if (!needToLoadLatestData && !needToLoadAllPasses)
{
//Log.LogInformation($"Segment load on {cellAddress} exiting as neither latest nor all passes required");
return(FileSystemErrorStatus.OK); // Nothing more to do here
}
// Lock the segment briefly while its contents is being loaded
lock (segment)
{
if (!(needToLoadLatestData ^ segment.HasLatestData) && !(needToLoadAllPasses ^ segment.HasAllPasses))
{
//Log.LogInformation($"Segment load on {cellAddress} leaving quietly as a previous thread has performed the load");
return(FileSystemErrorStatus.OK); // The load operation was performed on another thread. Leave quietly
}
// Ensure the appropriate storage is allocated
if (needToLoadLatestData)
{
segment.AllocateLatestPassGrid();
}
if (needToLoadAllPasses)
{
segment.AllocateFullPassStacks();
}
if (!segment.SegmentInfo.ExistsInPersistentStore)
{
//Log.LogInformation($"Segment load on {cellAddress} exiting as segment does not exist in persistent store");
return(FileSystemErrorStatus.OK); // Nothing more to do here
}
// Locate the segment file and load the data from it
var fullFileName = GetLeafSubGridSegmentFullFileName(cellAddress, segment.SegmentInfo);
// Load the cells into it from its file
var fsError = subGrid.LoadSegmentFromStorage(storageProxy, fullFileName, segment, needToLoadLatestData, needToLoadAllPasses);
if (fsError != FileSystemErrorStatus.OK)
{
//Log.LogInformation($"Segment load on {cellAddress} failed, performing allocation cleanup activities");
// Something bad happened. Remove the segment from the list. Return failure to the caller.
if (loadAllPasses)
{
segment.DeAllocateFullPassStacks();
}
if (loadLatestData)
{
segment.DeAllocateLatestPassGrid();
}
return(fsError);
}
}
//Log.LogInformation($"Segment load on {cellAddress} succeeded, AllPasses?={Segment.HasAllPasses}, Segment.PassesData?Null={Segment.PassesData==null} ");
return(FileSystemErrorStatus.OK);
}
private ISubGridCellPassesDataSegment LocateNextSubGridSegmentInIteration()
{
ISubGridCellPassesDataSegment result = null;
if (IterationState.SubGrid == null)
{
Log.LogCritical("No sub grid node assigned to iteration state");
return(null);
}
while (IterationState.NextSegment())
{
var segmentInfo = IterationState.Directory.SegmentDirectory[IterationState.Idx];
if (segmentInfo.Segment != null)
{
result = segmentInfo.Segment;
}
// If there is no segment present in the cache then it can't be dirty, so is
// not a candidate to be returned by the iterator
// Similarly if the caller is only interested in segments that are present in the cache,
// we do not need to read it from the persistent store
if (!ReturnDirtyOnly && !ReturnCachedItemsOnly)
{
// This additional check to determine if the segment is defined
// is necessary to check if an earlier thread through this code has
// already allocated the new segment
if (segmentInfo.Segment == null)
{
IterationState.SubGrid.AllocateSegment(segmentInfo);
}
result = segmentInfo.Segment;
if (result == null)
{
throw new TRexSubGridProcessingException("IterationState.SubGrid.Cells.AllocateSegment failed to create a new segment");
}
}
if (result != null)
{
if (!result.Dirty && ReturnDirtyOnly)
{
// The segment is not dirty, and the iterator has been instructed only to return
// dirty segments, so ignore this one
result = null;
continue;
}
if (!result.Dirty && !ReturnCachedItemsOnly &&
(RetrieveAllPasses && !result.HasAllPasses || RetrieveLatestData && !result.HasLatestData))
{
var fsResult = ((IServerSubGridTree)IterationState.SubGrid.Owner).LoadLeafSubGridSegment
(StorageProxyForSubGridSegments,
new SubGridCellAddress(IterationState.SubGrid.OriginX, IterationState.SubGrid.OriginY),
RetrieveLatestData, RetrieveAllPasses,
IterationState.SubGrid,
result);
if (fsResult == FileSystemErrorStatus.OK)
{
// TRex has no separate cache - it is in Ignite
}
else
{
// TRex has no separate cache - it is in Ignite
// Segment failed to be loaded. Multiple messages will have been posted to the log.
// Move to the next item in the iteration
// Specific FS failures indicate corruption in the data store that should preclude further iteration and
// processir of the contents of this sub grid. These conditions result in the sub grid being blacklisted
// and the iterator returning no further information for this subgrid
if (fsResult == FileSystemErrorStatus.GranuleDoesNotExist)
{
SegmentIterationBlackListed = true;
Log.LogWarning($"Black listing segment iteration due to file system failure {fsResult} for sub grid {IterationState.SubGrid.Moniker()}");
}
result = null;
continue;
}
}
}
if (result != null) // We have a candidate to return as the next item in the iteration
{
break;
}
}
return(result);
}
