Ejemplo n.º 1
0
        public void Test_SubgridSegment_Cleaver()
        {
            // Create a sub grid to hold the segment
            IServerLeafSubGrid subGrid = MakeSubgridWith10240CellPassesAtOneSecondIntervals();

            // Exercise the cleaver!
            // Instruct the segment container to cleave the segment
            // Set the segment to not dirty - it should be ignored
            subGrid.Cells.PassesData[0].Dirty = false;

            var cleaver = new SubGridSegmentCleaver();

            cleaver.PerformSegmentCleaving(StorageProxy.Instance(StorageMutability.Mutable), subGrid, 10000);

            Assert.True(1 == subGrid.Cells.PassesData.Count, $"After cleaving with no dirty segments there are {subGrid.Cells.PassesData.Count} segments instead of the expected one segments");

            // Set the segment to not dirty - it should be ignored
            subGrid.Cells.PassesData[0].Dirty = true;
            cleaver = new SubGridSegmentCleaver();
            cleaver.PerformSegmentCleaving(StorageProxy.Instance(StorageMutability.Mutable), subGrid, 10000);

            //Check there are now two segments in total
            Assert.True(2 == subGrid.Cells.PassesData.Count, $"After cleaving there are {subGrid.Cells.PassesData.Count} segments instead of the expected two segments");

            //Check the total number of passes across the two segments is 10240, and the maximum pass count is 5
            ISubGridCellPassesDataSegment segment1 = subGrid.Cells.PassesData[0];
            ISubGridCellPassesDataSegment segment2 = subGrid.Cells.PassesData[1];

            segment1.PassesData.CalculateTotalPasses(out int totalPassCount1, out _, out int maximumPassCount1);
            segment2.PassesData.CalculateTotalPasses(out int totalPassCount2, out _, out int maximumPassCount2);

            Assert.True(10240 == (totalPassCount1 + totalPassCount2), $"Totals ({totalPassCount1} and {totalPassCount2} don't add up to 10240 after cleaving");
            Assert.True(5 == maximumPassCount1, $"Maximum pass count 1 {maximumPassCount1}, is not 5");
            Assert.True(5 == maximumPassCount2, $"Maximum pass count 2 {maximumPassCount2}, is not 5");

            // Check the segment pass count in the segment is correct
            Assert.True(totalPassCount1 == segment1.PassesData.SegmentPassCount, $"Total passes for segment 1 {totalPassCount1} is not equal to segmentPassCount in that segment {segment1.PassesData.SegmentPassCount}");
            Assert.True(totalPassCount2 == segment2.PassesData.SegmentPassCount, $"Total passes for segment 2 {totalPassCount2} is not equal to segmentPassCount in that segment {segment2.PassesData.SegmentPassCount}");
        }
Ejemplo n.º 2
0
        /// <summary>
        /// Orchestrates all the activities relating to saving the state of a created or modified sub grid, including
        /// cleaving, saving updated elements, creating new elements and arranging for the retirement of
        /// elements that have been replaced in the persistent store as a result of this activity.
        /// </summary>
        public bool SaveLeafSubGrid(IServerLeafSubGrid subGrid,
                                    IStorageProxy storageProxyForSubGrids,
                                    IStorageProxy storageProxyForSubGridSegments,
                                    List <ISubGridSpatialAffinityKey> invalidatedSpatialStreams)
        {
            //Log.LogInformation($"Saving {subGrid.Moniker()} to persistent store");

            try
            {
                // Perform segment cleaving as the first activity in the action of saving a leaf sub grid
                // to disk. This reduces the number of segment cleaving actions that would otherwise
                // be performed in the context of the aggregated integrator.

                if (_segmentCleavingOperationsToLog)
                {
                    _log.LogDebug($"About to perform segment cleaving on {subGrid.Moniker()}");
                }

                var cleaver = new SubGridSegmentCleaver();

                cleaver.PerformSegmentCleaving(storageProxyForSubGridSegments, subGrid);

                // Calculate the cell last pass information here, immediately before it is
                // committed to the persistent store. The reason for this is to remove this
                // compute intensive operation from the critical path in TAG file processing
                // (which is the only writer of this information in the Raptor system).
                // The computer is instructed to do a partial recompute, which will recompute
                // all segments from the first segment marked as dirty.
                subGrid.ComputeLatestPassInformation(false, storageProxyForSubGridSegments);

                if (_segmentCleavingOperationsToLog)
                {
                    _log.LogInformation($"SaveLeafSubGrid: {subGrid.Moniker()} ({subGrid.Cells.PassesData.Count} segments)");
                }

                var modifiedOriginalSegments = new List <ISubGridCellPassesDataSegment>(100);
                var originAddress            = new SubGridCellAddress(subGrid.OriginX, subGrid.OriginY);

                // The following used to be an assert/exception. However, this is may readily
                // happen if there are no modified segments resulting from processing a
                // process TAG file where the Dirty flag for the sub grid is set but no cell
                // passes are added to segments in that sub grid. As this is not an integrity
                // issue the persistence of the modified sub grid is allowed, but noted in
                // the log for posterity.
                if (subGrid.Cells.PassesData.Count == 0)
                {
                    _log.LogInformation(
                        $"Note: Saving a sub grid, {subGrid.Moniker()}, (Segments = {subGrid.Cells.PassesData.Count}, Dirty = {subGrid.Dirty}) with no cached sub grid segments to the persistent store in SaveLeafSubGrid (possible reprocessing of TAG file with no cell pass changes). " +
                        $"SubGrid.Directory.PersistedClovenSegments.Count={cleaver.PersistedClovenSegments?.Count}, ModifiedOriginalFiles.Count={modifiedOriginalSegments.Count}, NewSegmentsFromCleaving.Count={cleaver.NewSegmentsFromCleaving.Count}");
                }

                var iterator = new SubGridSegmentIterator(subGrid, storageProxyForSubGridSegments)
                {
                    IterationDirection = IterationDirection.Forwards,
                    ReturnDirtyOnly    = true,
                };

                //**********************************************************************
                //***Construct list of original segment files that have been modified***
                //*** These files may be updated in-situ with no sub grid/segment    ***
                //*** integrity issues wrt the segment directory in the sub grid     ***
                //**********************************************************************

                iterator.MoveToFirstSubGridSegment();
                while (iterator.CurrentSubGridSegment != null)
                {
                    if (iterator.CurrentSubGridSegment.SegmentInfo.ExistsInPersistentStore && iterator.CurrentSubGridSegment.Dirty)
                    {
                        modifiedOriginalSegments.Add(iterator.CurrentSubGridSegment);
                    }
                    iterator.MoveToNextSubGridSegment();
                }

                //**********************************************************************
                //*** Construct list of spatial streams that will be deleted or replaced
                //*** in the FS file. These will be passed to the call that saves the
                //*** sub grid directory file as an instruction to place them into the
                //*** deferred deletion list
                //**********************************************************************

                lock (invalidatedSpatialStreams)
                {
                    if (cleaver.PersistedClovenSegments != null)
                    {
                        invalidatedSpatialStreams.AddRange(cleaver.PersistedClovenSegments);
                    }

                    invalidatedSpatialStreams.AddRange(modifiedOriginalSegments.Select(x => x.SegmentInfo.AffinityKey(ID)));
                }

                if (cleaver.NewSegmentsFromCleaving.Count > 0)
                {
                    //**********************************************************************
                    //***    Write new segment files generated by cleaving               ***
                    //*** File system integrity failures here will have no effect on the ***
                    //*** sub grid/segment directory as they are not referenced by it.   ***
                    //*** At worst they become orphans they may be cleaned by in the FS  ***
                    //*** recovery phase                                                 ***
                    //**********************************************************************

                    if (_segmentCleavingOperationsToLog)
                    {
                        _log.LogInformation($"Sub grid has {cleaver.NewSegmentsFromCleaving.Count} new segments from cleaving");
                    }

                    foreach (var segment in cleaver.NewSegmentsFromCleaving)
                    {
                        // Update the version of the segment as it is about to be written
                        segment.SegmentInfo.Touch();

                        segment.SaveToFile(storageProxyForSubGridSegments, GetLeafSubGridSegmentFullFileName(originAddress, segment.SegmentInfo), out var fsError);
                        segment.Dirty = false;

                        if (fsError == FileSystemErrorStatus.OK)
                        {
                            if (_segmentCleavingOperationsToLog)
                            {
                                _log.LogInformation($"Saved new cloven grid segment file: {GetLeafSubGridSegmentFullFileName(originAddress, segment.SegmentInfo)}");
                            }
                        }
                        else
                        {
                            _log.LogWarning($"Failed to save cloven grid segment {GetLeafSubGridSegmentFullFileName(originAddress, segment.SegmentInfo)}: Error:{fsError}");
                            return(false);
                        }
                    }
                }

                if (modifiedOriginalSegments.Count > 0)
                {
                    //**********************************************************************
                    //***    Write modified segment files                                ***
                    //*** File system integrity failures here will have no effect on the ***
                    //*** sub grid/segment directory as the previous version of the      ***
                    //*** modified file being written will be recovered.                 ***
                    //**********************************************************************

                    if (_log.IsTraceEnabled())
                    {
                        _log.LogTrace($"Sub grid has {modifiedOriginalSegments.Count} modified segments");
                    }

                    foreach (var segment in modifiedOriginalSegments)
                    {
                        // Update the version of the segment as it is about to be written
                        segment.SegmentInfo.Touch();
                        segment.Dirty = false;

                        if (segment.SaveToFile(storageProxyForSubGridSegments, GetLeafSubGridSegmentFullFileName(originAddress, segment.SegmentInfo), out var fsError))
                        {
                            segment.Dirty = false;
                            if (_log.IsTraceEnabled())
                            {
                                _log.LogTrace($"Saved modified grid segment file: {segment}");
                            }
                        }
                        else
                        {
                            _log.LogError($"Failed to save modified original grid segment {GetLeafSubGridSegmentFullFileName(originAddress, segment.SegmentInfo)}: Error:{fsError}");
                            return(false);
                        }
                    }
                }

                // Any remaining dirty segments in the sub grid will be due to previously empty sub grids with newly
                // created segments from ingest processing that have not required cleaving. These segments do not require any
                // special treatment are jsut saved to persistent store

                foreach (var segment in subGrid.Directory.SegmentDirectory.Select(x => x.Segment).Where(x => (x?.Dirty ?? false)))
                {
                    if (segment.SaveToFile(storageProxyForSubGridSegments, GetLeafSubGridSegmentFullFileName(originAddress, segment.SegmentInfo), out var fsError))
                    {
                        segment.Dirty = false;
                        if (_log.IsTraceEnabled())
                        {
                            _log.LogTrace($"Saved new sub grid segment file: {segment}");
                        }
                    }
                    else
                    {
                        _log.LogError($"Failed to save new sub grid segment {GetLeafSubGridSegmentFullFileName(originAddress, segment.SegmentInfo)}: Error:{fsError}");
                        return(false);
                    }
                }

                //**********************************************************************
                //***                 Write the sub grid directory file              ***
                //**********************************************************************

                /*
                 * There is no need to add the sub grid directory stream to the segment retirement
                 * queue as this will be automatically replaced when the new version of the
                 * sub grid directory is written to persistent store.
                 *
                 * // Add the stream representing the sub grid directory file to the list of
                 * // invalidated streams as this stream will be replaced with the stream
                 * // containing the updated directory information. Note: This only needs to
                 * // be done if the sub grid has previously been read from the FS file (if not
                 * // it has been created and not yet persisted to the store.
                 *
                 * if (subGrid.Directory.ExistsInPersistentStore)
                 * {
                 * // Include an additional invalidated spatial stream for the sub grid directory stream
                 * lock (invalidatedSpatialStreams)
                 * {
                 *  invalidatedSpatialStreams.Add(subGrid.AffinityKey());
                 * }
                 * }
                 */

                if (subGrid.SaveDirectoryToFile(storageProxyForSubGrids, GetLeafSubGridFullFileName(originAddress)))
                {
                    if (_log.IsTraceEnabled())
                    {
                        _log.LogTrace($"Saved grid directory file: {GetLeafSubGridFullFileName(originAddress)}");
                    }
                }
                else
                {
                    if (_log.IsTraceEnabled())
                    {
                        _log.LogTrace($"Failed to save grid: {GetLeafSubGridFullFileName(originAddress)}");
                    }
                    return(false);
                }

                //**********************************************************************
                //***                   Reset segment dirty flags                    ***
                //**********************************************************************

                iterator.MoveToFirstSubGridSegment();
                while (iterator.CurrentSubGridSegment != null)
                {
                    iterator.CurrentSubGridSegment.Dirty = false;
                    iterator.CurrentSubGridSegment.SegmentInfo.ExistsInPersistentStore = true;

                    iterator.MoveToNextSubGridSegment();
                }

                //Log.LogInformation($"Completed saving {subGrid.Moniker()} to persistent store");

                return(true);
            }
            catch (Exception e)
            {
                _log.LogError(e, "Exception raised in SaveLeafSubGrid");
            }

            return(false);
        }