public SubGridCellPassesDataSegmentInfo(DateTime startTime, DateTime endTime, ISubGridCellPassesDataSegment segment) : this() { StartTime = startTime; EndTime = endTime; Segment = segment; }
public void MetaTest_Test_MakeSubgridWith10240CellPassesAtOneSecondIntervals() { // Create a subgrid to hold the segment IServerLeafSubGrid subGrid = MakeSubgridWith10240CellPassesAtOneSecondIntervals(); ISubGridCellPassesDataSegment segment = subGrid.Cells.PassesData[0]; // Check all cells have exactly 10 passes VSS.TRex.SubGridTrees.Core.Utilities.SubGridUtilities.SubGridDimensionalIterator((x, y) => { Assert.True(segment.PassesData.PassCount(x, y) == 10, $"Cell in segment at {x},{y} does not have 10 cell passes"); }); //Check the total number of passes is 10240, and the maximum pass count is 10 segment.PassesData.CalculateTotalPasses(out int totalPassCount, out _, out int maximumPassCount); Assert.True(10240 == totalPassCount, "Initial total pass count not 10240"); Assert.True(10240 == segment.PassesData.SegmentPassCount, $"segment.PassesData.SegmentPassCount does not equal 10240 (it is {segment.PassesData.SegmentPassCount})"); Assert.True(10 == maximumPassCount, "Initial maximum pass count not 10"); // Check the time range is as expected segment.PassesData.CalculateTimeRange(out DateTime startSegmentTime, out DateTime endSegmentTime); Assert.True(endSegmentTime > startSegmentTime, $"End time {endSegmentTime} not greater than startTime {startSegmentTime}"); Assert.True(startSegmentTime == startTime, $"Start time {startSegmentTime} not equal to {startTime} as expected"); Assert.True(endSegmentTime == startTime.AddSeconds(10239), $"End time {endSegmentTime} not equal to {startTime.AddSeconds(10239)} as expected"); }
public int Add(ISubGridCellPassesDataSegment item) { int Index = Count - 1; while (Index >= 0 && item.SegmentInfo.StartTime < Items[Index].SegmentInfo.StartTime) { Index--; } Index++; Items.Insert(Index, item); return(Index); /* * {$IFDEF DEBUG} * Counter := 0; * for Index := 0 to Count - 2 do * if Items[Index].SegmentInfo.StartTime >= Items[Index + 1].SegmentInfo.StartTime then * begin * SIGLogMessage.PublishNoODS(Self, Format('Segment passes list out of order %.6f versus %.6f. Segment count = %d', { SKIP} * [Items[Index].SegmentInfo.StartTime, Items[Index + 1].SegmentInfo.StartTime, Count]), slmcAssert); * Inc(Counter); * end; * if Counter > 0 then * DumpSegmentsToLog; * {$ENDIF} */ }
public void Test_SubgridSegment_Cleaver() { // Create a sub grid to hold the segment IServerLeafSubGrid subGrid = MakeSubgridWith10240CellPassesAtOneSecondIntervals(); // Exercise the cleaver! // Instruct the segment container to cleave the segment // Set the segment to not dirty - it should be ignored subGrid.Cells.PassesData[0].Dirty = false; var cleaver = new SubGridSegmentCleaver(); cleaver.PerformSegmentCleaving(StorageProxy.Instance(StorageMutability.Mutable), subGrid, 10000); Assert.True(1 == subGrid.Cells.PassesData.Count, $"After cleaving with no dirty segments there are {subGrid.Cells.PassesData.Count} segments instead of the expected one segments"); // Set the segment to not dirty - it should be ignored subGrid.Cells.PassesData[0].Dirty = true; cleaver = new SubGridSegmentCleaver(); cleaver.PerformSegmentCleaving(StorageProxy.Instance(StorageMutability.Mutable), subGrid, 10000); //Check there are now two segments in total Assert.True(2 == subGrid.Cells.PassesData.Count, $"After cleaving there are {subGrid.Cells.PassesData.Count} segments instead of the expected two segments"); //Check the total number of passes across the two segments is 10240, and the maximum pass count is 5 ISubGridCellPassesDataSegment segment1 = subGrid.Cells.PassesData[0]; ISubGridCellPassesDataSegment segment2 = subGrid.Cells.PassesData[1]; segment1.PassesData.CalculateTotalPasses(out int totalPassCount1, out _, out int maximumPassCount1); segment2.PassesData.CalculateTotalPasses(out int totalPassCount2, out _, out int maximumPassCount2); Assert.True(10240 == (totalPassCount1 + totalPassCount2), $"Totals ({totalPassCount1} and {totalPassCount2} don't add up to 10240 after cleaving"); Assert.True(5 == maximumPassCount1, $"Maximum pass count 1 {maximumPassCount1}, is not 5"); Assert.True(5 == maximumPassCount2, $"Maximum pass count 2 {maximumPassCount2}, is not 5"); // Check the segment pass count in the segment is correct Assert.True(totalPassCount1 == segment1.PassesData.SegmentPassCount, $"Total passes for segment 1 {totalPassCount1} is not equal to segmentPassCount in that segment {segment1.PassesData.SegmentPassCount}"); Assert.True(totalPassCount2 == segment2.PassesData.SegmentPassCount, $"Total passes for segment 2 {totalPassCount2} is not equal to segmentPassCount in that segment {segment2.PassesData.SegmentPassCount}"); }
public void Test_SubgridSegmentCleaving() { // Create a sub grid to hold the segment. the subGrid retains the limit in the subGrid constructor in MakeSubgridWith10240CellPassesAtOneSecondIntervals() IServerLeafSubGrid subGrid = MakeSubgridWith10240CellPassesAtOneSecondIntervals(); ISubGridCellPassesDataSegment segment = subGrid.Cells.PassesData[0]; // Instruct the segment container to cleave the segment with a limit of 100000 var newSegmentsFromCleaving = new List <ISubGridCellPassesDataSegment>(); var persistedClovenSegments = new List <ISubGridSpatialAffinityKey>(); Assert.False(subGrid.Cells.CleaveSegment(segment, newSegmentsFromCleaving, persistedClovenSegments, 100000), "Segment was cloven when cell pass count was below limit"); // Set the cleaving limit to 10000 to force the segment TO BE cloven = the cleave result should be true subGrid = MakeSubgridWith10240CellPassesAtOneSecondIntervals(); segment = subGrid.Cells.PassesData[0]; newSegmentsFromCleaving = new List <ISubGridCellPassesDataSegment>(); persistedClovenSegments = new List <ISubGridSpatialAffinityKey>(); Assert.True(subGrid.Cells.CleaveSegment(segment, newSegmentsFromCleaving, persistedClovenSegments, 10000), "Segment failed to cleave with pass count above limit"); //Check there are now two segments in total Assert.True(2 == subGrid.Cells.PassesData.Count, $"After cleaving there are {subGrid.Cells.PassesData.Count} segments instead of the expected two segments"); //Check the total number of passes across the two segments is 10240, and the maximum pass count is 5 ISubGridCellPassesDataSegment segment1 = subGrid.Cells.PassesData[0]; ISubGridCellPassesDataSegment segment2 = subGrid.Cells.PassesData[1]; segment1.PassesData.CalculateTotalPasses(out int totalPassCount1, out _, out int maximumPassCount1); segment2.PassesData.CalculateTotalPasses(out int totalPassCount2, out _, out int maximumPassCount2); Assert.True(10240 == totalPassCount1 + totalPassCount2, $"Totals ({totalPassCount1} and {totalPassCount2} don't add up to 10240 after cleaving"); Assert.True(5 == maximumPassCount1, $"Maximum pass count 1 {maximumPassCount1}, is not 5"); Assert.True(5 == maximumPassCount2, $"Maximum pass count 2 {maximumPassCount2}, is not 5"); // Check the segment pass count in the segment is correct Assert.True(totalPassCount1 == segment1.PassesData.SegmentPassCount, $"Total passes for segment 1 {totalPassCount1} is not equal to segmentPassCount in that segment {segment1.PassesData.SegmentPassCount}"); Assert.True(totalPassCount2 == segment2.PassesData.SegmentPassCount, $"Total passes for segment 2 {totalPassCount2} is not equal to segmentPassCount in that segment {segment2.PassesData.SegmentPassCount}"); }
public FileSystemErrorStatus LoadLeafSubGridSegment(IStorageProxy storageProxy, SubGridCellAddress cellAddress, bool loadLatestData, bool loadAllPasses, IServerLeafSubGrid subGrid, ISubGridCellPassesDataSegment segment) { //Log.LogInformation($"Segment load on {cellAddress}:{Segment.SegmentInfo.StartTime}-{Segment.SegmentInfo.EndTime} beginning, loadLatestData = {loadLatestData}, loadAllPasses = {loadAllPasses}"); var needToLoadLatestData = loadLatestData && !segment.HasLatestData; var needToLoadAllPasses = loadAllPasses && !segment.HasAllPasses; if (!needToLoadLatestData && !needToLoadAllPasses) { //Log.LogInformation($"Segment load on {cellAddress} exiting as neither latest nor all passes required"); return(FileSystemErrorStatus.OK); // Nothing more to do here } // Lock the segment briefly while its contents is being loaded lock (segment) { if (!(needToLoadLatestData ^ segment.HasLatestData) && !(needToLoadAllPasses ^ segment.HasAllPasses)) { //Log.LogInformation($"Segment load on {cellAddress} leaving quietly as a previous thread has performed the load"); return(FileSystemErrorStatus.OK); // The load operation was performed on another thread. Leave quietly } // Ensure the appropriate storage is allocated if (needToLoadLatestData) { segment.AllocateLatestPassGrid(); } if (needToLoadAllPasses) { segment.AllocateFullPassStacks(); } if (!segment.SegmentInfo.ExistsInPersistentStore) { //Log.LogInformation($"Segment load on {cellAddress} exiting as segment does not exist in persistent store"); return(FileSystemErrorStatus.OK); // Nothing more to do here } // Locate the segment file and load the data from it var fullFileName = GetLeafSubGridSegmentFullFileName(cellAddress, segment.SegmentInfo); // Load the cells into it from its file var fsError = subGrid.LoadSegmentFromStorage(storageProxy, fullFileName, segment, needToLoadLatestData, needToLoadAllPasses); if (fsError != FileSystemErrorStatus.OK) { //Log.LogInformation($"Segment load on {cellAddress} failed, performing allocation cleanup activities"); // Something bad happened. Remove the segment from the list. Return failure to the caller. if (loadAllPasses) { segment.DeAllocateFullPassStacks(); } if (loadLatestData) { segment.DeAllocateLatestPassGrid(); } return(fsError); } } //Log.LogInformation($"Segment load on {cellAddress} succeeded, AllPasses?={Segment.HasAllPasses}, Segment.PassesData?Null={Segment.PassesData==null} "); return(FileSystemErrorStatus.OK); }
private ISubGridCellPassesDataSegment LocateNextSubGridSegmentInIteration() { ISubGridCellPassesDataSegment result = null; if (IterationState.SubGrid == null) { Log.LogCritical("No sub grid node assigned to iteration state"); return(null); } while (IterationState.NextSegment()) { var segmentInfo = IterationState.Directory.SegmentDirectory[IterationState.Idx]; if (segmentInfo.Segment != null) { result = segmentInfo.Segment; } // If there is no segment present in the cache then it can't be dirty, so is // not a candidate to be returned by the iterator // Similarly if the caller is only interested in segments that are present in the cache, // we do not need to read it from the persistent store if (!ReturnDirtyOnly && !ReturnCachedItemsOnly) { // This additional check to determine if the segment is defined // is necessary to check if an earlier thread through this code has // already allocated the new segment if (segmentInfo.Segment == null) { IterationState.SubGrid.AllocateSegment(segmentInfo); } result = segmentInfo.Segment; if (result == null) { throw new TRexSubGridProcessingException("IterationState.SubGrid.Cells.AllocateSegment failed to create a new segment"); } } if (result != null) { if (!result.Dirty && ReturnDirtyOnly) { // The segment is not dirty, and the iterator has been instructed only to return // dirty segments, so ignore this one result = null; continue; } if (!result.Dirty && !ReturnCachedItemsOnly && (RetrieveAllPasses && !result.HasAllPasses || RetrieveLatestData && !result.HasLatestData)) { var fsResult = ((IServerSubGridTree)IterationState.SubGrid.Owner).LoadLeafSubGridSegment (StorageProxyForSubGridSegments, new SubGridCellAddress(IterationState.SubGrid.OriginX, IterationState.SubGrid.OriginY), RetrieveLatestData, RetrieveAllPasses, IterationState.SubGrid, result); if (fsResult == FileSystemErrorStatus.OK) { // TRex has no separate cache - it is in Ignite } else { // TRex has no separate cache - it is in Ignite // Segment failed to be loaded. Multiple messages will have been posted to the log. // Move to the next item in the iteration // Specific FS failures indicate corruption in the data store that should preclude further iteration and // processir of the contents of this sub grid. These conditions result in the sub grid being blacklisted // and the iterator returning no further information for this subgrid if (fsResult == FileSystemErrorStatus.GranuleDoesNotExist) { SegmentIterationBlackListed = true; Log.LogWarning($"Black listing segment iteration due to file system failure {fsResult} for sub grid {IterationState.SubGrid.Moniker()}"); } result = null; continue; } } } if (result != null) // We have a candidate to return as the next item in the iteration { break; } } return(result); }