IEnumerable <ChunkInfo> GetAllLatestChunksExceptLast(TFChunkEnumerator chunkEnumerator, int lastChunkNum) { foreach (var chunkInfo in chunkEnumerator.EnumerateChunks(lastChunkNum)) { switch (chunkInfo) { case LatestVersion(var fileName, var start, _): if (start <= lastChunkNum - 1) { yield return new ChunkInfo { ChunkFileName = fileName, ChunkStartNumber = start } } ; break; case MissingVersion(var fileName, var start): if (start <= lastChunkNum - 1) { throw new CorruptDatabaseException(new ChunkNotFoundException(fileName)); } break; } } }
private static void EnsureNoExcessiveChunks(TFChunkEnumerator chunkEnumerator, int lastChunkNum) { var extraneousFiles = new List <string>(); foreach (var chunkInfo in chunkEnumerator.EnumerateChunks(lastChunkNum)) { switch (chunkInfo) { case LatestVersion(var fileName, var start, _): if (start > lastChunkNum) { extraneousFiles.Add(fileName); } break; case OldVersion(var fileName, var start): if (start > lastChunkNum) { extraneousFiles.Add(fileName); } break; } } if (!extraneousFiles.IsEmpty()) { throw new CorruptDatabaseException(new ExtraneousFileFoundException( $"Unexpected files: {string.Join(", ", extraneousFiles)}.")); } }
private void RemoveOldChunksVersions(TFChunkEnumerator chunkEnumerator, int lastChunkNum) { foreach (var chunkInfo in chunkEnumerator.EnumerateChunks(lastChunkNum)) { switch (chunkInfo) { case OldVersion(var fileName, var start): if (start <= lastChunkNum) { RemoveFile("Removing old chunk version: {chunk}...", fileName); } break; } } }
public void TruncateDb(long truncateChk) { var writerChk = _config.WriterCheckpoint.Read(); var requestedTruncation = writerChk - truncateChk; if (_config.MaxTruncation >= 0 && requestedTruncation > _config.MaxTruncation) { Log.Error( "MaxTruncation is set and truncate checkpoint is out of bounds. MaxTruncation {maxTruncation} vs requested truncation {requestedTruncation} [{writerChk} => {truncateChk}]. To proceed, set MaxTruncation to -1 (no max) or greater than {requestedTruncationHint}.", _config.MaxTruncation, requestedTruncation, writerChk, truncateChk, requestedTruncation); throw new Exception( string.Format("MaxTruncation is set ({0}) and truncate checkpoint is out of bounds (requested truncation is {1} [{2} => {3}]).", _config.MaxTruncation, requestedTruncation, writerChk, truncateChk)); } var oldLastChunkNum = (int)(writerChk / _config.ChunkSize); var newLastChunkNum = (int)(truncateChk / _config.ChunkSize); var chunkEnumerator = new TFChunkEnumerator(_config.FileNamingStrategy); var excessiveChunks = _config.FileNamingStrategy.GetAllVersionsFor(oldLastChunkNum + 1); if (excessiveChunks.Length > 0) { throw new Exception(string.Format("During truncation of DB excessive TFChunks were found:\n{0}.", string.Join("\n", excessiveChunks))); } ChunkHeader newLastChunkHeader = null; string newLastChunkFilename = null; // find the chunk to truncate to foreach (var chunkInfo in chunkEnumerator.EnumerateChunks(oldLastChunkNum)) { switch (chunkInfo) { case LatestVersion(var fileName, var _, var end): if (newLastChunkFilename != null || end < newLastChunkNum) { break; } newLastChunkHeader = ReadChunkHeader(fileName); newLastChunkFilename = fileName; break; case MissingVersion(var fileName, var chunkNum): if (chunkNum < newLastChunkNum) { throw new Exception($"Could not find any chunk #{fileName}."); } break; } } // it's not bad if there is no file, it could have been deleted on previous run if (newLastChunkHeader != null) { var chunksToDelete = new List <string>(); var chunkNumToDeleteFrom = newLastChunkNum + 1; if (newLastChunkHeader.IsScavenged) { Log.Information( "Deleting ALL chunks from #{chunkStartNumber} inclusively " + "as truncation position is in the middle of scavenged chunk.", newLastChunkHeader.ChunkStartNumber); chunkNumToDeleteFrom = newLastChunkHeader.ChunkStartNumber; } foreach (var chunkInfo in chunkEnumerator.EnumerateChunks(oldLastChunkNum)) { switch (chunkInfo) { case LatestVersion(var fileName, var start, _): if (start >= chunkNumToDeleteFrom) { chunksToDelete.Add(fileName); } break; case OldVersion(var fileName, var start): if (start >= chunkNumToDeleteFrom) { chunksToDelete.Add(fileName); } break; } } // we need to remove excessive chunks from largest number to lowest one, so in case of crash // mid-process, we don't end up with broken non-sequential chunks sequence. chunksToDelete.Reverse(); foreach (var chunkFile in chunksToDelete) { Log.Information("File {chunk} will be deleted during TruncateDb procedure.", chunkFile); File.SetAttributes(chunkFile, FileAttributes.Normal); File.Delete(chunkFile); } if (!newLastChunkHeader.IsScavenged) { TruncateChunkAndFillWithZeros(newLastChunkHeader, newLastChunkFilename, truncateChk); } else { truncateChk = newLastChunkHeader.ChunkStartPosition; Log.Information( "Setting TruncateCheckpoint to {truncateCheckpoint} " + "as truncation position is in the middle of scavenged chunk.", truncateChk); } } if (_config.EpochCheckpoint.Read() >= truncateChk) { var epochChk = _config.EpochCheckpoint.Read(); Log.Information("Truncating epoch from {epochFrom} (0x{epochFrom:X}) to {epochTo} (0x{epochTo:X}).", epochChk, epochChk, -1, -1); _config.EpochCheckpoint.Write(-1); _config.EpochCheckpoint.Flush(); } if (_config.ChaserCheckpoint.Read() > truncateChk) { var chaserChk = _config.ChaserCheckpoint.Read(); Log.Information( "Truncating chaser from {chaserCheckpoint} (0x{chaserCheckpoint:X}) to {truncateCheckpoint} (0x{truncateCheckpoint:X}).", chaserChk, chaserChk, truncateChk, truncateChk); _config.ChaserCheckpoint.Write(truncateChk); _config.ChaserCheckpoint.Flush(); } if (_config.WriterCheckpoint.Read() > truncateChk) { var writerCheckpoint = _config.WriterCheckpoint.Read(); Log.Information( "Truncating writer from {writerCheckpoint} (0x{writerCheckpoint:X}) to {truncateCheckpoint} (0x{truncateCheckpoint:X}).", writerCheckpoint, writerCheckpoint, truncateChk, truncateChk); _config.WriterCheckpoint.Write(truncateChk); _config.WriterCheckpoint.Flush(); } Log.Information("Resetting TruncateCheckpoint to {truncateCheckpoint} (0x{truncateCheckpoint:X}).", -1, -1); _config.TruncateCheckpoint.Write(-1); _config.TruncateCheckpoint.Flush(); }
public void Open(bool verifyHash = true, bool readOnly = false, int threads = 1) { Ensure.Positive(threads, "threads"); ValidateReaderChecksumsMustBeLess(Config); var checkpoint = Config.WriterCheckpoint.Read(); if (Config.InMemDb) { Manager.AddNewChunk(); return; } var lastChunkNum = (int)(checkpoint / Config.ChunkSize); var lastChunkVersions = Config.FileNamingStrategy.GetAllVersionsFor(lastChunkNum); var chunkEnumerator = new TFChunkEnumerator(Config.FileNamingStrategy); try { Parallel.ForEach(GetAllLatestChunksExceptLast(chunkEnumerator, lastChunkNum), // the last chunk is dealt with separately new ParallelOptions { MaxDegreeOfParallelism = threads }, chunkInfo => { TFChunk.TFChunk chunk; if (lastChunkVersions.Length == 0 && (chunkInfo.ChunkStartNumber + 1) * (long)Config.ChunkSize == checkpoint) { // The situation where the logical data size is exactly divisible by ChunkSize, // so it might happen that we have checkpoint indicating one more chunk should exist, // but the actual last chunk is (lastChunkNum-1) one and it could be not completed yet -- perfectly valid situation. var footer = ReadChunkFooter(chunkInfo.ChunkFileName); if (footer.IsCompleted) { chunk = TFChunk.TFChunk.FromCompletedFile(chunkInfo.ChunkFileName, verifyHash: false, unbufferedRead: Config.Unbuffered, initialReaderCount: Config.InitialReaderCount, maxReaderCount: Config.MaxReaderCount, optimizeReadSideCache: Config.OptimizeReadSideCache, reduceFileCachePressure: Config.ReduceFileCachePressure); } else { chunk = TFChunk.TFChunk.FromOngoingFile(chunkInfo.ChunkFileName, Config.ChunkSize, checkSize: false, unbuffered: Config.Unbuffered, writethrough: Config.WriteThrough, initialReaderCount: Config.InitialReaderCount, maxReaderCount: Config.MaxReaderCount, reduceFileCachePressure: Config.ReduceFileCachePressure); // chunk is full with data, we should complete it right here if (!readOnly) { chunk.Complete(); } } } else { chunk = TFChunk.TFChunk.FromCompletedFile(chunkInfo.ChunkFileName, verifyHash: false, unbufferedRead: Config.Unbuffered, initialReaderCount: Config.InitialReaderCount, maxReaderCount: Config.MaxReaderCount, optimizeReadSideCache: Config.OptimizeReadSideCache, reduceFileCachePressure: Config.ReduceFileCachePressure); } // This call is theadsafe. Manager.AddChunk(chunk); }); } catch (AggregateException aggEx) { // We only really care that *something* is wrong - throw the first inner exception. throw aggEx.InnerException; } if (lastChunkVersions.Length == 0) { var onBoundary = checkpoint == (Config.ChunkSize * (long)lastChunkNum); if (!onBoundary) { throw new CorruptDatabaseException( new ChunkNotFoundException(Config.FileNamingStrategy.GetFilenameFor(lastChunkNum, 0))); } if (!readOnly) { Manager.AddNewChunk(); } } else { var chunkFileName = lastChunkVersions[0]; var chunkHeader = ReadChunkHeader(chunkFileName); var chunkLocalPos = chunkHeader.GetLocalLogPosition(checkpoint); if (chunkHeader.IsScavenged) { var lastChunk = TFChunk.TFChunk.FromCompletedFile(chunkFileName, verifyHash: false, unbufferedRead: Config.Unbuffered, initialReaderCount: Config.InitialReaderCount, maxReaderCount: Config.MaxReaderCount, optimizeReadSideCache: Config.OptimizeReadSideCache, reduceFileCachePressure: Config.ReduceFileCachePressure); if (lastChunk.ChunkFooter.LogicalDataSize != chunkLocalPos) { lastChunk.Dispose(); throw new CorruptDatabaseException(new BadChunkInDatabaseException( string.Format("Chunk {0} is corrupted. Expected local chunk position: {1}, " + "but Chunk.LogicalDataSize is {2} (Chunk.PhysicalDataSize is {3}). Writer checkpoint: {4}.", chunkFileName, chunkLocalPos, lastChunk.LogicalDataSize, lastChunk.PhysicalDataSize, checkpoint))); } Manager.AddChunk(lastChunk); if (!readOnly) { _log.Information( "Moving WriterCheckpoint from {checkpoint} to {chunkEndPosition}, as it points to the scavenged chunk. " + "If that was not caused by replication of scavenged chunks, that could be a bug.", checkpoint, lastChunk.ChunkHeader.ChunkEndPosition); Config.WriterCheckpoint.Write(lastChunk.ChunkHeader.ChunkEndPosition); Config.WriterCheckpoint.Flush(); Manager.AddNewChunk(); } } else { var lastChunk = TFChunk.TFChunk.FromOngoingFile(chunkFileName, (int)chunkLocalPos, checkSize: false, unbuffered: Config.Unbuffered, writethrough: Config.WriteThrough, initialReaderCount: Config.InitialReaderCount, maxReaderCount: Config.MaxReaderCount, reduceFileCachePressure: Config.ReduceFileCachePressure); Manager.AddChunk(lastChunk); } } _log.Information("Ensuring no excessive chunks..."); EnsureNoExcessiveChunks(chunkEnumerator, lastChunkNum); _log.Information("Done ensuring no excessive chunks."); if (!readOnly) { _log.Information("Removing old chunk versions..."); RemoveOldChunksVersions(chunkEnumerator, lastChunkNum); _log.Information("Done removing old chunk versions."); _log.Information("Cleaning up temp files..."); CleanUpTempFiles(); _log.Information("Done cleaning up temp files."); } if (verifyHash && lastChunkNum > 0) { var preLastChunk = Manager.GetChunk(lastChunkNum - 1); var lastBgChunkNum = preLastChunk.ChunkHeader.ChunkStartNumber; ThreadPool.QueueUserWorkItem(_ => { for (int chunkNum = lastBgChunkNum; chunkNum >= 0;) { var chunk = Manager.GetChunk(chunkNum); try { chunk.VerifyFileHash(); } catch (FileBeingDeletedException exc) { _log.Debug( "{exceptionType} exception was thrown while doing background validation of chunk {chunk}.", exc.GetType().Name, chunk); _log.Debug( "That's probably OK, especially if truncation was request at the same time: {e}.", exc.Message); } catch (Exception exc) { _log.Fatal(exc, "Verification of chunk {chunk} failed, terminating server...", chunk); var msg = string.Format("Verification of chunk {0} failed, terminating server...", chunk); Application.Exit(ExitCode.Error, msg); return; } chunkNum = chunk.ChunkHeader.ChunkStartNumber - 1; } }); } Manager.EnableCaching(); }