/// <summary> /// Cleaves all dirty segments requiring cleaving within the given sub grid /// </summary> public void PerformSegmentCleaving(IStorageProxy storageProxyForSubGridSegments, IServerLeafSubGrid subGrid, int subGridSegmentPassCountLimit = 0) { var iterator = new SubGridSegmentIterator(subGrid, storageProxyForSubGridSegments) { IterationDirection = IterationDirection.Forwards, ReturnDirtyOnly = true, RetrieveAllPasses = true }; var origin = new SubGridCellAddress(subGrid.OriginX, subGrid.OriginY); if (!iterator.MoveToFirstSubGridSegment()) { return; } do { var segment = iterator.CurrentSubGridSegment; var cleavedTimeRangeStart = segment.SegmentInfo.StartTime; var cleavedTimeRangeEnd = segment.SegmentInfo.EndTime; if (!segment.RequiresCleaving(out var totalPassCount, out var maximumPassCount)) { continue; } if (subGrid.Cells.CleaveSegment(segment, NewSegmentsFromCleaving, PersistedClovenSegments, subGridSegmentPassCountLimit)) { iterator.SegmentListExtended(); if (_segmentCleavingOperationsToLog) { _log.LogInformation( $"Info: Performed cleave on segment ({cleavedTimeRangeStart}-{cleavedTimeRangeEnd}) of sub grid {ServerSubGridTree.GetLeafSubGridFullFileName(origin)}. TotalPassCount = {totalPassCount} MaximumPassCount = {maximumPassCount}"); } } else { // The segment cleave failed. While this is not a serious problem (as the sub grid will be // cleaved at some point in the future when it is modified again via tag file processing etc) // it will be noted in the log. _log.LogWarning( $"Cleave on segment ({cleavedTimeRangeStart}-{cleavedTimeRangeEnd}) of sub grid {ServerSubGridTree.GetLeafSubGridFullFileName(origin)} failed. TotalPassCount = {totalPassCount} MaximumPassCount = {maximumPassCount}"); } if (_segmentCleavingOperationsToLog) { if (segment.RequiresCleaving(out totalPassCount, out maximumPassCount)) { _log.LogWarning( $"Cleave on segment ({cleavedTimeRangeStart}-{cleavedTimeRangeEnd}) of sub grid {subGrid.Moniker()} failed to reduce cell pass count below maximums (max passes = {totalPassCount}/{subGridSegmentPassCountLimit}, per cell = {maximumPassCount}/{_subGridMaxSegmentCellPassesLimit})"); } } } while (iterator.MoveToNextSubGridSegment()); }
/// <summary> /// Orchestrates all the activities relating to saving the state of a created or modified sub grid, including /// cleaving, saving updated elements, creating new elements and arranging for the retirement of /// elements that have been replaced in the persistent store as a result of this activity. /// </summary> public bool SaveLeafSubGrid(IServerLeafSubGrid subGrid, IStorageProxy storageProxyForSubGrids, IStorageProxy storageProxyForSubGridSegments, List <ISubGridSpatialAffinityKey> invalidatedSpatialStreams) { //Log.LogInformation($"Saving {subGrid.Moniker()} to persistent store"); try { // Perform segment cleaving as the first activity in the action of saving a leaf sub grid // to disk. This reduces the number of segment cleaving actions that would otherwise // be performed in the context of the aggregated integrator. if (_segmentCleavingOperationsToLog) { _log.LogDebug($"About to perform segment cleaving on {subGrid.Moniker()}"); } var cleaver = new SubGridSegmentCleaver(); cleaver.PerformSegmentCleaving(storageProxyForSubGridSegments, subGrid); // Calculate the cell last pass information here, immediately before it is // committed to the persistent store. The reason for this is to remove this // compute intensive operation from the critical path in TAG file processing // (which is the only writer of this information in the Raptor system). // The computer is instructed to do a partial recompute, which will recompute // all segments from the first segment marked as dirty. subGrid.ComputeLatestPassInformation(false, storageProxyForSubGridSegments); if (_segmentCleavingOperationsToLog) { _log.LogInformation($"SaveLeafSubGrid: {subGrid.Moniker()} ({subGrid.Cells.PassesData.Count} segments)"); } var modifiedOriginalSegments = new List <ISubGridCellPassesDataSegment>(100); var originAddress = new SubGridCellAddress(subGrid.OriginX, subGrid.OriginY); // The following used to be an assert/exception. However, this is may readily // happen if there are no modified segments resulting from processing a // process TAG file where the Dirty flag for the sub grid is set but no cell // passes are added to segments in that sub grid. As this is not an integrity // issue the persistence of the modified sub grid is allowed, but noted in // the log for posterity. if (subGrid.Cells.PassesData.Count == 0) { _log.LogInformation( $"Note: Saving a sub grid, {subGrid.Moniker()}, (Segments = {subGrid.Cells.PassesData.Count}, Dirty = {subGrid.Dirty}) with no cached sub grid segments to the persistent store in SaveLeafSubGrid (possible reprocessing of TAG file with no cell pass changes). " + $"SubGrid.Directory.PersistedClovenSegments.Count={cleaver.PersistedClovenSegments?.Count}, ModifiedOriginalFiles.Count={modifiedOriginalSegments.Count}, NewSegmentsFromCleaving.Count={cleaver.NewSegmentsFromCleaving.Count}"); } var iterator = new SubGridSegmentIterator(subGrid, storageProxyForSubGridSegments) { IterationDirection = IterationDirection.Forwards, ReturnDirtyOnly = true, }; //********************************************************************** //***Construct list of original segment files that have been modified*** //*** These files may be updated in-situ with no sub grid/segment *** //*** integrity issues wrt the segment directory in the sub grid *** //********************************************************************** iterator.MoveToFirstSubGridSegment(); while (iterator.CurrentSubGridSegment != null) { if (iterator.CurrentSubGridSegment.SegmentInfo.ExistsInPersistentStore && iterator.CurrentSubGridSegment.Dirty) { modifiedOriginalSegments.Add(iterator.CurrentSubGridSegment); } iterator.MoveToNextSubGridSegment(); } //********************************************************************** //*** Construct list of spatial streams that will be deleted or replaced //*** in the FS file. These will be passed to the call that saves the //*** sub grid directory file as an instruction to place them into the //*** deferred deletion list //********************************************************************** lock (invalidatedSpatialStreams) { if (cleaver.PersistedClovenSegments != null) { invalidatedSpatialStreams.AddRange(cleaver.PersistedClovenSegments); } invalidatedSpatialStreams.AddRange(modifiedOriginalSegments.Select(x => x.SegmentInfo.AffinityKey(ID))); } if (cleaver.NewSegmentsFromCleaving.Count > 0) { //********************************************************************** //*** Write new segment files generated by cleaving *** //*** File system integrity failures here will have no effect on the *** //*** sub grid/segment directory as they are not referenced by it. *** //*** At worst they become orphans they may be cleaned by in the FS *** //*** recovery phase *** //********************************************************************** if (_segmentCleavingOperationsToLog) { _log.LogInformation($"Sub grid has {cleaver.NewSegmentsFromCleaving.Count} new segments from cleaving"); } foreach (var segment in cleaver.NewSegmentsFromCleaving) { // Update the version of the segment as it is about to be written segment.SegmentInfo.Touch(); segment.SaveToFile(storageProxyForSubGridSegments, GetLeafSubGridSegmentFullFileName(originAddress, segment.SegmentInfo), out var fsError); segment.Dirty = false; if (fsError == FileSystemErrorStatus.OK) { if (_segmentCleavingOperationsToLog) { _log.LogInformation($"Saved new cloven grid segment file: {GetLeafSubGridSegmentFullFileName(originAddress, segment.SegmentInfo)}"); } } else { _log.LogWarning($"Failed to save cloven grid segment {GetLeafSubGridSegmentFullFileName(originAddress, segment.SegmentInfo)}: Error:{fsError}"); return(false); } } } if (modifiedOriginalSegments.Count > 0) { //********************************************************************** //*** Write modified segment files *** //*** File system integrity failures here will have no effect on the *** //*** sub grid/segment directory as the previous version of the *** //*** modified file being written will be recovered. *** //********************************************************************** if (_log.IsTraceEnabled()) { _log.LogTrace($"Sub grid has {modifiedOriginalSegments.Count} modified segments"); } foreach (var segment in modifiedOriginalSegments) { // Update the version of the segment as it is about to be written segment.SegmentInfo.Touch(); segment.Dirty = false; if (segment.SaveToFile(storageProxyForSubGridSegments, GetLeafSubGridSegmentFullFileName(originAddress, segment.SegmentInfo), out var fsError)) { segment.Dirty = false; if (_log.IsTraceEnabled()) { _log.LogTrace($"Saved modified grid segment file: {segment}"); } } else { _log.LogError($"Failed to save modified original grid segment {GetLeafSubGridSegmentFullFileName(originAddress, segment.SegmentInfo)}: Error:{fsError}"); return(false); } } } // Any remaining dirty segments in the sub grid will be due to previously empty sub grids with newly // created segments from ingest processing that have not required cleaving. These segments do not require any // special treatment are jsut saved to persistent store foreach (var segment in subGrid.Directory.SegmentDirectory.Select(x => x.Segment).Where(x => (x?.Dirty ?? false))) { if (segment.SaveToFile(storageProxyForSubGridSegments, GetLeafSubGridSegmentFullFileName(originAddress, segment.SegmentInfo), out var fsError)) { segment.Dirty = false; if (_log.IsTraceEnabled()) { _log.LogTrace($"Saved new sub grid segment file: {segment}"); } } else { _log.LogError($"Failed to save new sub grid segment {GetLeafSubGridSegmentFullFileName(originAddress, segment.SegmentInfo)}: Error:{fsError}"); return(false); } } //********************************************************************** //*** Write the sub grid directory file *** //********************************************************************** /* * There is no need to add the sub grid directory stream to the segment retirement * queue as this will be automatically replaced when the new version of the * sub grid directory is written to persistent store. * * // Add the stream representing the sub grid directory file to the list of * // invalidated streams as this stream will be replaced with the stream * // containing the updated directory information. Note: This only needs to * // be done if the sub grid has previously been read from the FS file (if not * // it has been created and not yet persisted to the store. * * if (subGrid.Directory.ExistsInPersistentStore) * { * // Include an additional invalidated spatial stream for the sub grid directory stream * lock (invalidatedSpatialStreams) * { * invalidatedSpatialStreams.Add(subGrid.AffinityKey()); * } * } */ if (subGrid.SaveDirectoryToFile(storageProxyForSubGrids, GetLeafSubGridFullFileName(originAddress))) { if (_log.IsTraceEnabled()) { _log.LogTrace($"Saved grid directory file: {GetLeafSubGridFullFileName(originAddress)}"); } } else { if (_log.IsTraceEnabled()) { _log.LogTrace($"Failed to save grid: {GetLeafSubGridFullFileName(originAddress)}"); } return(false); } //********************************************************************** //*** Reset segment dirty flags *** //********************************************************************** iterator.MoveToFirstSubGridSegment(); while (iterator.CurrentSubGridSegment != null) { iterator.CurrentSubGridSegment.Dirty = false; iterator.CurrentSubGridSegment.SegmentInfo.ExistsInPersistentStore = true; iterator.MoveToNextSubGridSegment(); } //Log.LogInformation($"Completed saving {subGrid.Moniker()} to persistent store"); return(true); } catch (Exception e) { _log.LogError(e, "Exception raised in SaveLeafSubGrid"); } return(false); }