IEnumerable <ChunkInfo> GetAllLatestChunksExceptLast(TFChunkEnumerator chunkEnumerator, int lastChunkNum)
{
foreach (var chunkInfo in chunkEnumerator.EnumerateChunks(lastChunkNum))
{
switch (chunkInfo)
{
case LatestVersion(var fileName, var start, _):
if (start <= lastChunkNum - 1)
{
yield return new ChunkInfo {
ChunkFileName = fileName, ChunkStartNumber = start
}
}
;
break;
case MissingVersion(var fileName, var start):
if (start <= lastChunkNum - 1)
{
throw new CorruptDatabaseException(new ChunkNotFoundException(fileName));
}
break;
}
}
}
private static void EnsureNoExcessiveChunks(TFChunkEnumerator chunkEnumerator, int lastChunkNum)
{
var extraneousFiles = new List <string>();
foreach (var chunkInfo in chunkEnumerator.EnumerateChunks(lastChunkNum))
{
switch (chunkInfo)
{
case LatestVersion(var fileName, var start, _):
if (start > lastChunkNum)
{
extraneousFiles.Add(fileName);
}
break;
case OldVersion(var fileName, var start):
if (start > lastChunkNum)
{
extraneousFiles.Add(fileName);
}
break;
}
}
if (!extraneousFiles.IsEmpty())
{
throw new CorruptDatabaseException(new ExtraneousFileFoundException(
$"Unexpected files: {string.Join(", ", extraneousFiles)}."));
}
}
private void RemoveOldChunksVersions(TFChunkEnumerator chunkEnumerator, int lastChunkNum)
{
foreach (var chunkInfo in chunkEnumerator.EnumerateChunks(lastChunkNum))
{
switch (chunkInfo)
{
case OldVersion(var fileName, var start):
if (start <= lastChunkNum)
{
RemoveFile("Removing old chunk version: {chunk}...", fileName);
}
break;
}
}
}
public void TruncateDb(long truncateChk)
{
var writerChk = _config.WriterCheckpoint.Read();
var requestedTruncation = writerChk - truncateChk;
if (_config.MaxTruncation >= 0 && requestedTruncation > _config.MaxTruncation)
{
Log.Error(
"MaxTruncation is set and truncate checkpoint is out of bounds. MaxTruncation {maxTruncation} vs requested truncation {requestedTruncation} [{writerChk} => {truncateChk}]. To proceed, set MaxTruncation to -1 (no max) or greater than {requestedTruncationHint}.",
_config.MaxTruncation, requestedTruncation, writerChk, truncateChk, requestedTruncation);
throw new Exception(
string.Format("MaxTruncation is set ({0}) and truncate checkpoint is out of bounds (requested truncation is {1} [{2} => {3}]).", _config.MaxTruncation, requestedTruncation, writerChk, truncateChk));
}
var oldLastChunkNum = (int)(writerChk / _config.ChunkSize);
var newLastChunkNum = (int)(truncateChk / _config.ChunkSize);
var chunkEnumerator = new TFChunkEnumerator(_config.FileNamingStrategy);
var excessiveChunks = _config.FileNamingStrategy.GetAllVersionsFor(oldLastChunkNum + 1);
if (excessiveChunks.Length > 0)
{
throw new Exception(string.Format("During truncation of DB excessive TFChunks were found:\n{0}.",
string.Join("\n", excessiveChunks)));
}
ChunkHeader newLastChunkHeader = null;
string newLastChunkFilename = null;
// find the chunk to truncate to
foreach (var chunkInfo in chunkEnumerator.EnumerateChunks(oldLastChunkNum))
{
switch (chunkInfo)
{
case LatestVersion(var fileName, var _, var end):
if (newLastChunkFilename != null || end < newLastChunkNum)
{
break;
}
newLastChunkHeader = ReadChunkHeader(fileName);
newLastChunkFilename = fileName;
break;
case MissingVersion(var fileName, var chunkNum):
if (chunkNum < newLastChunkNum)
{
throw new Exception($"Could not find any chunk #{fileName}.");
}
break;
}
}
// it's not bad if there is no file, it could have been deleted on previous run
if (newLastChunkHeader != null)
{
var chunksToDelete = new List <string>();
var chunkNumToDeleteFrom = newLastChunkNum + 1;
if (newLastChunkHeader.IsScavenged)
{
Log.Information(
"Deleting ALL chunks from #{chunkStartNumber} inclusively "
+ "as truncation position is in the middle of scavenged chunk.",
newLastChunkHeader.ChunkStartNumber);
chunkNumToDeleteFrom = newLastChunkHeader.ChunkStartNumber;
}
foreach (var chunkInfo in chunkEnumerator.EnumerateChunks(oldLastChunkNum))
{
switch (chunkInfo)
{
case LatestVersion(var fileName, var start, _):
if (start >= chunkNumToDeleteFrom)
{
chunksToDelete.Add(fileName);
}
break;
case OldVersion(var fileName, var start):
if (start >= chunkNumToDeleteFrom)
{
chunksToDelete.Add(fileName);
}
break;
}
}
// we need to remove excessive chunks from largest number to lowest one, so in case of crash
// mid-process, we don't end up with broken non-sequential chunks sequence.
chunksToDelete.Reverse();
foreach (var chunkFile in chunksToDelete)
{
Log.Information("File {chunk} will be deleted during TruncateDb procedure.", chunkFile);
File.SetAttributes(chunkFile, FileAttributes.Normal);
File.Delete(chunkFile);
}
if (!newLastChunkHeader.IsScavenged)
{
TruncateChunkAndFillWithZeros(newLastChunkHeader, newLastChunkFilename, truncateChk);
}
else
{
truncateChk = newLastChunkHeader.ChunkStartPosition;
Log.Information(
"Setting TruncateCheckpoint to {truncateCheckpoint} "
+ "as truncation position is in the middle of scavenged chunk.",
truncateChk);
}
}
if (_config.EpochCheckpoint.Read() >= truncateChk)
{
var epochChk = _config.EpochCheckpoint.Read();
Log.Information("Truncating epoch from {epochFrom} (0x{epochFrom:X}) to {epochTo} (0x{epochTo:X}).", epochChk,
epochChk, -1, -1);
_config.EpochCheckpoint.Write(-1);
_config.EpochCheckpoint.Flush();
}
if (_config.ChaserCheckpoint.Read() > truncateChk)
{
var chaserChk = _config.ChaserCheckpoint.Read();
Log.Information(
"Truncating chaser from {chaserCheckpoint} (0x{chaserCheckpoint:X}) to {truncateCheckpoint} (0x{truncateCheckpoint:X}).",
chaserChk, chaserChk, truncateChk, truncateChk);
_config.ChaserCheckpoint.Write(truncateChk);
_config.ChaserCheckpoint.Flush();
}
if (_config.WriterCheckpoint.Read() > truncateChk)
{
var writerCheckpoint = _config.WriterCheckpoint.Read();
Log.Information(
"Truncating writer from {writerCheckpoint} (0x{writerCheckpoint:X}) to {truncateCheckpoint} (0x{truncateCheckpoint:X}).",
writerCheckpoint, writerCheckpoint, truncateChk, truncateChk);
_config.WriterCheckpoint.Write(truncateChk);
_config.WriterCheckpoint.Flush();
}
Log.Information("Resetting TruncateCheckpoint to {truncateCheckpoint} (0x{truncateCheckpoint:X}).", -1, -1);
_config.TruncateCheckpoint.Write(-1);
_config.TruncateCheckpoint.Flush();
}
public void Open(bool verifyHash = true, bool readOnly = false, int threads = 1)
{
Ensure.Positive(threads, "threads");
ValidateReaderChecksumsMustBeLess(Config);
var checkpoint = Config.WriterCheckpoint.Read();
if (Config.InMemDb)
{
Manager.AddNewChunk();
return;
}
var lastChunkNum = (int)(checkpoint / Config.ChunkSize);
var lastChunkVersions = Config.FileNamingStrategy.GetAllVersionsFor(lastChunkNum);
var chunkEnumerator = new TFChunkEnumerator(Config.FileNamingStrategy);
try {
Parallel.ForEach(GetAllLatestChunksExceptLast(chunkEnumerator, lastChunkNum), // the last chunk is dealt with separately
new ParallelOptions {
MaxDegreeOfParallelism = threads
},
chunkInfo => {
TFChunk.TFChunk chunk;
if (lastChunkVersions.Length == 0 &&
(chunkInfo.ChunkStartNumber + 1) * (long)Config.ChunkSize == checkpoint)
{
// The situation where the logical data size is exactly divisible by ChunkSize,
// so it might happen that we have checkpoint indicating one more chunk should exist,
// but the actual last chunk is (lastChunkNum-1) one and it could be not completed yet -- perfectly valid situation.
var footer = ReadChunkFooter(chunkInfo.ChunkFileName);
if (footer.IsCompleted)
{
chunk = TFChunk.TFChunk.FromCompletedFile(chunkInfo.ChunkFileName, verifyHash: false,
unbufferedRead: Config.Unbuffered,
initialReaderCount: Config.InitialReaderCount,
maxReaderCount: Config.MaxReaderCount,
optimizeReadSideCache: Config.OptimizeReadSideCache,
reduceFileCachePressure: Config.ReduceFileCachePressure);
}
else
{
chunk = TFChunk.TFChunk.FromOngoingFile(chunkInfo.ChunkFileName, Config.ChunkSize,
checkSize: false,
unbuffered: Config.Unbuffered,
writethrough: Config.WriteThrough, initialReaderCount: Config.InitialReaderCount,
maxReaderCount: Config.MaxReaderCount,
reduceFileCachePressure: Config.ReduceFileCachePressure);
// chunk is full with data, we should complete it right here
if (!readOnly)
{
chunk.Complete();
}
}
}
else
{
chunk = TFChunk.TFChunk.FromCompletedFile(chunkInfo.ChunkFileName, verifyHash: false,
unbufferedRead: Config.Unbuffered,
initialReaderCount: Config.InitialReaderCount,
maxReaderCount: Config.MaxReaderCount,
optimizeReadSideCache: Config.OptimizeReadSideCache,
reduceFileCachePressure: Config.ReduceFileCachePressure);
}
// This call is theadsafe.
Manager.AddChunk(chunk);
});
} catch (AggregateException aggEx) {
// We only really care that *something* is wrong - throw the first inner exception.
throw aggEx.InnerException;
}
if (lastChunkVersions.Length == 0)
{
var onBoundary = checkpoint == (Config.ChunkSize * (long)lastChunkNum);
if (!onBoundary)
{
throw new CorruptDatabaseException(
new ChunkNotFoundException(Config.FileNamingStrategy.GetFilenameFor(lastChunkNum, 0)));
}
if (!readOnly)
{
Manager.AddNewChunk();
}
}
else
{
var chunkFileName = lastChunkVersions[0];
var chunkHeader = ReadChunkHeader(chunkFileName);
var chunkLocalPos = chunkHeader.GetLocalLogPosition(checkpoint);
if (chunkHeader.IsScavenged)
{
var lastChunk = TFChunk.TFChunk.FromCompletedFile(chunkFileName, verifyHash: false,
unbufferedRead: Config.Unbuffered,
initialReaderCount: Config.InitialReaderCount,
maxReaderCount: Config.MaxReaderCount,
optimizeReadSideCache: Config.OptimizeReadSideCache,
reduceFileCachePressure: Config.ReduceFileCachePressure);
if (lastChunk.ChunkFooter.LogicalDataSize != chunkLocalPos)
{
lastChunk.Dispose();
throw new CorruptDatabaseException(new BadChunkInDatabaseException(
string.Format("Chunk {0} is corrupted. Expected local chunk position: {1}, "
+ "but Chunk.LogicalDataSize is {2} (Chunk.PhysicalDataSize is {3}). Writer checkpoint: {4}.",
chunkFileName, chunkLocalPos, lastChunk.LogicalDataSize, lastChunk.PhysicalDataSize,
checkpoint)));
}
Manager.AddChunk(lastChunk);
if (!readOnly)
{
_log.Information(
"Moving WriterCheckpoint from {checkpoint} to {chunkEndPosition}, as it points to the scavenged chunk. "
+ "If that was not caused by replication of scavenged chunks, that could be a bug.",
checkpoint, lastChunk.ChunkHeader.ChunkEndPosition);
Config.WriterCheckpoint.Write(lastChunk.ChunkHeader.ChunkEndPosition);
Config.WriterCheckpoint.Flush();
Manager.AddNewChunk();
}
}
else
{
var lastChunk = TFChunk.TFChunk.FromOngoingFile(chunkFileName, (int)chunkLocalPos, checkSize: false,
unbuffered: Config.Unbuffered,
writethrough: Config.WriteThrough, initialReaderCount: Config.InitialReaderCount,
maxReaderCount: Config.MaxReaderCount,
reduceFileCachePressure: Config.ReduceFileCachePressure);
Manager.AddChunk(lastChunk);
}
}
_log.Information("Ensuring no excessive chunks...");
EnsureNoExcessiveChunks(chunkEnumerator, lastChunkNum);
_log.Information("Done ensuring no excessive chunks.");
if (!readOnly)
{
_log.Information("Removing old chunk versions...");
RemoveOldChunksVersions(chunkEnumerator, lastChunkNum);
_log.Information("Done removing old chunk versions.");
_log.Information("Cleaning up temp files...");
CleanUpTempFiles();
_log.Information("Done cleaning up temp files.");
}
if (verifyHash && lastChunkNum > 0)
{
var preLastChunk = Manager.GetChunk(lastChunkNum - 1);
var lastBgChunkNum = preLastChunk.ChunkHeader.ChunkStartNumber;
ThreadPool.QueueUserWorkItem(_ => {
for (int chunkNum = lastBgChunkNum; chunkNum >= 0;)
{
var chunk = Manager.GetChunk(chunkNum);
try {
chunk.VerifyFileHash();
} catch (FileBeingDeletedException exc) {
_log.Debug(
"{exceptionType} exception was thrown while doing background validation of chunk {chunk}.",
exc.GetType().Name, chunk);
_log.Debug(
"That's probably OK, especially if truncation was request at the same time: {e}.",
exc.Message);
} catch (Exception exc) {
_log.Fatal(exc, "Verification of chunk {chunk} failed, terminating server...",
chunk);
var msg = string.Format("Verification of chunk {0} failed, terminating server...", chunk);
Application.Exit(ExitCode.Error, msg);
return;
}
chunkNum = chunk.ChunkHeader.ChunkStartNumber - 1;
}
});
}
Manager.EnableCaching();
}