internal bool Run() { if (_keyValueDB.FileCollection.GetCount() == 0) { return(false); } _root = _keyValueDB.OldestRoot; var dontTouchGeneration = _keyValueDB.GetGeneration(_root.TrLogFileId); var preserveKeyIndexKey = _keyValueDB.CalculatePreserveKeyIndexKeyFromKeyIndexInfos(_keyValueDB.BuildKeyIndexInfos()); var preserveKeyIndexGeneration = _keyValueDB.CalculatePreserveKeyIndexGeneration(preserveKeyIndexKey); if (preserveKeyIndexKey < uint.MaxValue) { var dontTouchGenerationDueToPreserve = -1L; var fileInfo = _keyValueDB.FileCollection.FileInfoByIdx(preserveKeyIndexKey) as IKeyIndex; if (fileInfo != null) { dontTouchGenerationDueToPreserve = fileInfo.Generation; dontTouchGenerationDueToPreserve = Math.Min(dontTouchGenerationDueToPreserve, _keyValueDB.GetGeneration(fileInfo.TrLogFileId)); } dontTouchGeneration = Math.Min(dontTouchGeneration, dontTouchGenerationDueToPreserve); } InitFileStats(dontTouchGeneration); CalculateFileUsefullness(_root, false); // Oldest root is not uptodate CalculateFileUsefullness(_keyValueDB.LastCommited, true); // Last commited is uptodate MarkTotallyUselessFilesAsUnknown(); var totalWaste = CalcTotalWaste(); _keyValueDB.Logger?.CompactionStart(totalWaste); if (IsWasteSmall(totalWaste)) { if (_keyValueDB.DistanceFromLastKeyIndex(_root) > (ulong)(_keyValueDB.MaxTrLogFileSize / 4)) { _keyValueDB.CreateIndexFile(_cancellation, preserveKeyIndexGeneration); } _keyValueDB.FileCollection.DeleteAllUnknownFiles(); return(false); } _cancellation.ThrowIfCancellationRequested(); uint valueFileId; var writer = _keyValueDB.StartPureValuesFile(out valueFileId); var toRemoveFileIds = new List <uint>(); _newPositionMap = new Dictionary <ulong, uint>(); while (true) { var wastefullFileId = FindMostWastefullFile(_keyValueDB.MaxTrLogFileSize - writer.GetCurrentPosition()); if (wastefullFileId == 0) { break; } MoveValuesContent(writer, wastefullFileId); _fileStats[wastefullFileId] = new FileStat(0); toRemoveFileIds.Add(wastefullFileId); } var valueFile = _keyValueDB.FileCollection.GetFile(valueFileId); valueFile.HardFlush(); valueFile.Truncate(); _keyValueDB.Logger?.CompactionCreatedPureValueFile(valueFileId, valueFile.GetSize()); var btreesCorrectInTransactionId = _keyValueDB.AtomicallyChangeBTree(root => root.RemappingIterate((uint oldFileId, uint oldOffset, out uint newFileId, out uint newOffset) => { newFileId = valueFileId; _cancellation.ThrowIfCancellationRequested(); return(_newPositionMap.TryGetValue(((ulong)oldFileId << 32) | oldOffset, out newOffset)); })); _keyValueDB.CreateIndexFile(_cancellation, preserveKeyIndexGeneration); if (_newPositionMap.Count == 0) { toRemoveFileIds.Add(valueFileId); } if (_keyValueDB.AreAllTransactionsBeforeFinished(btreesCorrectInTransactionId)) { _keyValueDB.MarkAsUnknown(toRemoveFileIds); } _keyValueDB.FileCollection.DeleteAllUnknownFiles(); return(true); }
internal bool Run() { if (_keyValueDB.FileCollection.GetCount() == 0) { return(false); } _root = _keyValueDB.OldestRoot; var dontTouchGeneration = _keyValueDB.GetGeneration(_root.TrLogFileId); var preserveKeyIndexKey = _keyValueDB.CalculatePreserveKeyIndexKeyFromKeyIndexInfos(_keyValueDB.BuildKeyIndexInfos()); var preserveKeyIndexGeneration = _keyValueDB.CalculatePreserveKeyIndexGeneration(preserveKeyIndexKey); InitFileStats(dontTouchGeneration); long[] usedFilesFromOldGenerations = null; if (preserveKeyIndexKey < uint.MaxValue) { var dontTouchGenerationDueToPreserve = -1L; if (_keyValueDB.FileCollection.FileInfoByIdx(preserveKeyIndexKey) is IKeyIndex fileInfo) { dontTouchGenerationDueToPreserve = fileInfo.Generation; dontTouchGenerationDueToPreserve = Math.Min(dontTouchGenerationDueToPreserve, _keyValueDB.GetGeneration(fileInfo.TrLogFileId)); if (fileInfo.UsedFilesInOlderGenerations == null) { _keyValueDB.LoadUsedFilesFromKeyIndex(preserveKeyIndexKey, fileInfo); } usedFilesFromOldGenerations = fileInfo.UsedFilesInOlderGenerations; } dontTouchGeneration = Math.Min(dontTouchGeneration, dontTouchGenerationDueToPreserve); } var lastCommited = _keyValueDB.LastCommited; if (_root != lastCommited) { ForbidDeleteOfFilesUsedByStillRunningOldTransaction(_root); } ForbidDeletePreservingHistory(dontTouchGeneration, usedFilesFromOldGenerations); CalculateFileUsefullness(lastCommited); MarkTotallyUselessFilesAsUnknown(); var totalWaste = CalcTotalWaste(); _keyValueDB.Logger?.CompactionStart(totalWaste); if (IsWasteSmall(totalWaste)) { if (_keyValueDB.DistanceFromLastKeyIndex(_root) > (ulong)(_keyValueDB.MaxTrLogFileSize / 4)) { _keyValueDB.CreateIndexFile(_cancellation, preserveKeyIndexGeneration); } _keyValueDB.FileCollection.DeleteAllUnknownFiles(); return(false); } long btreesCorrectInTransactionId; var toRemoveFileIds = new List <uint>(); do { _newPositionMap = new Dictionary <ulong, ulong>(); do { CompactOnePureValueFileIteration(toRemoveFileIds); totalWaste = CalcTotalWaste(); } while (_newPositionMap.Count * 50 / (1024 * 1024) < _keyValueDB.CompactorRamLimitInMb && !IsWasteSmall(totalWaste)); btreesCorrectInTransactionId = _keyValueDB.ReplaceBTreeValues(_cancellation, _newPositionMap); } while (!IsWasteSmall(totalWaste)); _keyValueDB.CreateIndexFile(_cancellation, preserveKeyIndexGeneration); if (_keyValueDB.AreAllTransactionsBeforeFinished(btreesCorrectInTransactionId)) { _keyValueDB.MarkAsUnknown(toRemoveFileIds); } _keyValueDB.FileCollection.DeleteAllUnknownFiles(); return(true); }
internal bool Run() { if (_keyValueDB.FileCollection.GetCount() == 0) { return(false); } _root = _keyValueDB.OldestRoot; var dontTouchGeneration = _keyValueDB.GetGeneration(_root.TrLogFileId); var preserveKeyIndexKey = _keyValueDB.CalculatePreserveKeyIndexKeyFromKeyIndexInfos(_keyValueDB.BuildKeyIndexInfos()); var preserveKeyIndexGeneration = _keyValueDB.CalculatePreserveKeyIndexGeneration(preserveKeyIndexKey); InitFileStats(dontTouchGeneration); long[] usedFilesFromOldGenerations = null; if (preserveKeyIndexKey < uint.MaxValue) { var dontTouchGenerationDueToPreserve = -1L; var fileInfo = _keyValueDB.FileCollection.FileInfoByIdx(preserveKeyIndexKey) as IKeyIndex; if (fileInfo != null) { dontTouchGenerationDueToPreserve = fileInfo.Generation; dontTouchGenerationDueToPreserve = Math.Min(dontTouchGenerationDueToPreserve, _keyValueDB.GetGeneration(fileInfo.TrLogFileId)); if (fileInfo.UsedFilesInOlderGenerations == null) { _keyValueDB.LoadUsedFilesFromKeyIndex(preserveKeyIndexKey, fileInfo); } usedFilesFromOldGenerations = fileInfo.UsedFilesInOlderGenerations; } dontTouchGeneration = Math.Min(dontTouchGeneration, dontTouchGenerationDueToPreserve); } var lastCommited = _keyValueDB.LastCommited; if (_root != lastCommited) { ForbidDeleteOfFilesUsedByStillRunningOldTransaction(_root); } ForbidDeletePreservingHistory(dontTouchGeneration, usedFilesFromOldGenerations); CalculateFileUsefullness(lastCommited); MarkTotallyUselessFilesAsUnknown(); var totalWaste = CalcTotalWaste(); _keyValueDB.Logger?.CompactionStart(totalWaste); if (IsWasteSmall(totalWaste)) { if (_keyValueDB.DistanceFromLastKeyIndex(_root) > (ulong)(_keyValueDB.MaxTrLogFileSize / 4)) { _keyValueDB.CreateIndexFile(_cancellation, preserveKeyIndexGeneration); } _keyValueDB.FileCollection.DeleteAllUnknownFiles(); return(false); } _cancellation.ThrowIfCancellationRequested(); uint valueFileId; var writer = _keyValueDB.StartPureValuesFile(out valueFileId); var toRemoveFileIds = new List <uint>(); _newPositionMap = new Dictionary <ulong, uint>(); while (true) { var wastefullFileId = FindMostWastefullFile(_keyValueDB.MaxTrLogFileSize - writer.GetCurrentPosition()); if (wastefullFileId == 0) { break; } MoveValuesContent(writer, wastefullFileId); if (_fileStats[wastefullFileId].IsFreeToDelete()) { toRemoveFileIds.Add(wastefullFileId); } _fileStats[wastefullFileId] = new FileStat(0); } var valueFile = _keyValueDB.FileCollection.GetFile(valueFileId); valueFile.HardFlush(); valueFile.Truncate(); _keyValueDB.Logger?.CompactionCreatedPureValueFile(valueFileId, valueFile.GetSize()); var btreesCorrectInTransactionId = _keyValueDB.ReplaceBTreeValues(_cancellation, valueFileId, _newPositionMap); _keyValueDB.CreateIndexFile(_cancellation, preserveKeyIndexGeneration); if (_newPositionMap.Count == 0) { toRemoveFileIds.Add(valueFileId); } if (_keyValueDB.AreAllTransactionsBeforeFinished(btreesCorrectInTransactionId)) { _keyValueDB.MarkAsUnknown(toRemoveFileIds); } _keyValueDB.FileCollection.DeleteAllUnknownFiles(); return(true); }