public void Flushed_journals_should_become_recyclable_files_after_sync(int seed)
{
RequireFileBasedPager();
var r = new Random(seed);
var bytes = new byte[1024];
for (int i = 0; i < 1000; i++)
{
using (var tx = Env.WriteTransaction())
{
r.NextBytes(bytes);
tx.CreateTree("items").Add($"item/{i}", new MemoryStream(bytes));
tx.Commit();
}
}
Env.FlushLogToDataFile();
using (var operation = new WriteAheadJournal.JournalApplicator.SyncOperation(Env.Journal.Applicator))
{
var result = operation.SyncDataFile();
Assert.True(result);
}
var journalPath = ((StorageEnvironmentOptions.DirectoryStorageEnvironmentOptions)Env.Options).JournalPath.FullPath;
Assert.True(SpinWait.SpinUntil(() => new DirectoryInfo(journalPath).GetFiles($"{StorageEnvironmentOptions.RecyclableJournalFileNamePrefix}*").Length > 0,
TimeSpan.FromSeconds(30)));
}
public void ShouldExplicitlyErrorThatTurningOnIncrementalBackupAfterInitializingTheStorageIsntAllowed()
{
RequireFileBasedPager();
var random = new Random();
var buffer = new byte[4000];
random.NextBytes(buffer);
for (int i = 0; i < 300; i++)
{
using (var tx = Env.WriteTransaction())
{
var tree = tx.CreateTree("foo");
tree.Add("items/" + i, new MemoryStream(buffer));
tx.Commit();
}
}
Env.FlushLogToDataFile();
using (var op = new WriteAheadJournal.JournalApplicator.SyncOperation(Env.Journal.Applicator))
{
op.SyncDataFile();
}
Env.Options.IncrementalBackupEnabled = true;
var exception = Assert.Throws <InvalidOperationException>(() => BackupMethods.Incremental.ToFile(Env, _incrementalBackupTestUtils.IncrementalBackupFile(0)));
Assert.Equal("The first incremental backup creation failed because the first journal file " + StorageEnvironmentOptions.JournalName(0) + " was not found. Did you turn on the incremental backup feature after initializing the storage? In order to create backups incrementally the storage must be created with IncrementalBackupEnabled option set to 'true'.", exception.Message);
}
private void SyncEnvironment(EnvSyncReq req)
{
var storageEnvironment = req.Env;
if (storageEnvironment == null || storageEnvironment.Disposed)
{
return;
}
if (storageEnvironment.LastSyncCounter > req.LastKnownSyncCounter)
{
return; // we already a sync after this was scheduled
}
try
{
using (var operation = new WriteAheadJournal.JournalApplicator.SyncOperation(storageEnvironment.Journal.Applicator))
{
operation.SyncDataFile();
}
}
catch (Exception e)
{
if (_log.IsOperationsEnabled)
{
_log.Operations($"Failed to sync data file for {storageEnvironment.Options.BasePath}", e);
}
storageEnvironment.Options.SetCatastrophicFailure(ExceptionDispatchInfo.Capture(e));
}
}
private void SyncEnvironment(EnvSyncReq req)
{
var storageEnvironment = req.Env;
if (storageEnvironment == null || storageEnvironment.Disposed || storageEnvironment.Options.ManualSyncing)
{
return;
}
try
{
using (var operation = new WriteAheadJournal.JournalApplicator.SyncOperation(storageEnvironment.Journal.Applicator))
{
operation.SyncDataFile();
}
}
catch (Exception e)
{
if (_log.IsOperationsEnabled)
{
_log.Operations($"Failed to sync data file for {storageEnvironment.Options.BasePath}", e);
}
storageEnvironment.Options.SetCatastrophicFailure(ExceptionDispatchInfo.Capture(e));
}
}
public void Should_limit_number_of_recycled_journals()
{
RequireFileBasedPager();
Assert.Equal(3, Env.Options.MaxNumberOfRecyclableJournals);
var r = new Random(1);
var bytes = new byte[1024];
for (int i = 0; i < 1000; i++)
{
using (var tx = Env.WriteTransaction())
{
r.NextBytes(bytes);
tx.CreateTree("items").Add($"item/{i}", new MemoryStream(bytes));
tx.Commit();
}
}
Env.FlushLogToDataFile();
using (var op = new WriteAheadJournal.JournalApplicator.SyncOperation(Env.Journal.Applicator))
{
op.SyncDataFile();
}
var journalPath = ((StorageEnvironmentOptions.DirectoryStorageEnvironmentOptions)Env.Options).JournalPath.FullPath;
Assert.Equal(3, new DirectoryInfo(journalPath).GetFiles($"{StorageEnvironmentOptions.RecyclableJournalFileNamePrefix}*").Length);
}
public static void Execute(StorageEnvironmentOptions srcOptions,
StorageEnvironmentOptions.DirectoryStorageEnvironmentOptions compactOptions,
Action<StorageCompactionProgress> progressReport = null,
CancellationToken token = default(CancellationToken))
{
if (srcOptions.IncrementalBackupEnabled)
throw new InvalidOperationException(CannotCompactBecauseOfIncrementalBackup);
long minimalCompactedDataFileSize;
srcOptions.ManualFlushing = true; // prevent from flushing during compaction - we shouldn't touch any source files
compactOptions.ManualFlushing = true; // let us flush manually during data copy
using (var existingEnv = new StorageEnvironment(srcOptions))
using (var compactedEnv = new StorageEnvironment(compactOptions))
{
CopyTrees(existingEnv, compactedEnv, progressReport, token);
compactedEnv.FlushLogToDataFile();
bool synced;
const int maxNumberOfRetries = 100;
var syncRetries = 0;
while (true)
{
token.ThrowIfCancellationRequested();
using (var op = new WriteAheadJournal.JournalApplicator.SyncOperation(compactedEnv.Journal.Applicator))
{
try
{
synced = op.SyncDataFile();
if (synced || ++syncRetries >= maxNumberOfRetries)
break;
Thread.Sleep(100);
}
catch (Exception e)
{
existingEnv.Options.SetCatastrophicFailure(ExceptionDispatchInfo.Capture(e));
throw;
}
}
}
if (synced)
compactedEnv.Journal.Applicator.DeleteCurrentAlreadyFlushedJournal();
minimalCompactedDataFileSize = compactedEnv.NextPageNumber * Constants.Storage.PageSize;
}
using (var compactedDataFile = new FileStream(compactOptions.BasePath.Combine(Constants.DatabaseFilename).FullPath, FileMode.Open, FileAccess.ReadWrite))
{
compactedDataFile.SetLength(minimalCompactedDataFileSize);
}
}
public unsafe void ShouldFailBecauseFirstValidTransactionIsTheOneWhichIsNotSynced()
{
RequireFileBasedPager();
using (var tx = Env.WriteTransaction())
{
tx.CreateTree("tree");
tx.Commit();
}
var random = new Random(1);
for (var i = 0; i < 3; i++)
{
var buffer = new byte[1000];
random.NextBytes(buffer);
using (var tx = Env.WriteTransaction())
{
for (int j = 0; j < 100; j++)
{
tx.CreateTree("tree").Add("a" + i.ToString() + j.ToString(), new MemoryStream(buffer));
}
tx.Commit();
}
}
var lastJournal = Env.Journal.GetCurrentJournalInfo().CurrentJournal;
// let's flush and sync
Env.FlushLogToDataFile();
using (var operation = new WriteAheadJournal.JournalApplicator.SyncOperation(Env.Journal.Applicator))
{
operation.SyncDataFile();
}
for (var i = 0; i < 2; i++)
{
var buffer = new byte[1000];
random.NextBytes(buffer);
using (var tx = Env.WriteTransaction())
{
for (int j = 0; j < 100; j++)
{
tx.CreateTree("tree").Add("b" + i.ToString() + j.ToString(), new MemoryStream(buffer));
}
tx.Commit();
}
}
StopDatabase();
CorruptJournal(lastJournal, sizeof(TransactionHeader) + 5, Constants.Size.Kilobyte * 4 * 6);
Assert.Throws <InvalidJournalException>(StartDatabase);
}
public void Should_recover_if_jounral_is_missing_but_IgnoreInvalidJournalErrors_is_set()
{
RequireFileBasedPager();
var r = new Random(1);
var bytes = new byte[512];
for (int i = 0; i < 10; i++)
{
using (var tx = Env.WriteTransaction())
{
Tree tree = tx.CreateTree("tree");
for (int j = 0; j < 100; j++)
{
r.NextBytes(bytes);
tree.Add(new string((char)j, 1000), bytes);
}
tx.Commit();
}
}
Env.FlushLogToDataFile();
using (var operation = new WriteAheadJournal.JournalApplicator.SyncOperation(Env.Journal.Applicator))
{
operation.SyncDataFile();
}
for (int i = 0; i < 10; i++)
{
using (var tx = Env.WriteTransaction())
{
Tree tree = tx.CreateTree("tree");
for (int j = 0; j < 100; j++)
{
r.NextBytes(bytes);
tree.Add(new string((char)j, 1000), bytes);
}
tx.Commit();
}
}
Env.FlushLogToDataFile();
StopDatabase();
var journalPath = ((StorageEnvironmentOptions.DirectoryStorageEnvironmentOptions)Env.Options).JournalPath.FullPath;
var firstJournal = new DirectoryInfo(journalPath).GetFiles("*.journal").OrderBy(x => x.Name).First();
File.Delete(firstJournal.FullName);
StartDatabase(); // should not throw InvalidJournalException because we explicitly set options.IgnoreInvalidJournalErrors = true;
}
private void UpdateDatabaseStateAfterSyncIfNewerSyncCalled(bool forceFail)
{
var value1 = new byte[4000];
new Random().NextBytes(value1);
using (var tx = Env.WriteTransaction())
{
var tree = tx.CreateTree("foo");
tree.Add("foo/0", new MemoryStream(value1));
tx.Commit();
}
Env.Journal.Applicator.TestingWait = new ManualResetEvent(false);
Env.FlushLogToDataFile();
var task = new Task(() =>
{
using (var op = new WriteAheadJournal.JournalApplicator.SyncOperation(Env.Journal.Applicator))
{
Env.Journal.Applicator.TestingDoNotWait = forceFail;
op.SyncDataFile();
}
});
task.Start();
// ReSharper disable once AccessToDisposedClosure
Assert.True(SpinWait.SpinUntil(() => Env.Journal.Applicator.TestingNowWaitingSignal, 2000));
using (var tx = Env.WriteTransaction())
{
var tree = tx.CreateTree("bar");
tree.Add("bar/0", new MemoryStream(value1));
tx.Commit();
}
Env.FlushLogToDataFile();
using (var op = new WriteAheadJournal.JournalApplicator.SyncOperation(Env.Journal.Applicator))
{
Env.Journal.Applicator.TestingDoNotWait = true;
op.SyncDataFile();
}
Env.Journal.Applicator.TestingWait.Set();
Env.Journal.Applicator.TestingWait.Dispose();
Assert.True(SpinWait.SpinUntil(() => task.IsCompletedSuccessfully, 2000));
if (Env.Journal.Applicator.TestingSkippedUpdateDatabaseStateAfterSync == false)
{
throw new InvalidOperationException("Race condition was not detected at UpdateDatabaseStateAfterSync");
}
}
public void WillNotRetainJournalsAfterSync()
{
RequireFileBasedPager();
Options.ManualFlushing = true;
Options.MaxLogFileSize = 1024 * 1024;
Options.ManualSyncing = true;
for (int X = 0; X < 3; X++)
{
using (var tx = Env.WriteTransaction())
{
var tree = tx.CreateTree("t");
for (int i = 0; i < 50_000; i++)
{
tree.Add(i.ToString() + "-" + X, Guid.NewGuid().ToByteArray());
}
tx.Commit();
}
}
Env.FlushLogToDataFile();
var journalsDir = Path.Combine(DataDir, "Journals");
Assert.NotEmpty(Directory.GetFiles(journalsDir, "*.journal"));
using (var tx = Env.WriteTransaction())
{
var tree = tx.CreateTree("t");
for (int i = 0; i < 50_000; i++)
{
tree.Add(i.ToString() + "-5", Guid.NewGuid().ToByteArray());
}
tx.Commit();
}
using (var op = new WriteAheadJournal.JournalApplicator.SyncOperation(Env.Journal.Applicator))
{
op.SyncDataFile();
}
Env.FlushLogToDataFile();
using (var op = new WriteAheadJournal.JournalApplicator.SyncOperation(Env.Journal.Applicator))
{
op.SyncDataFile();
}
Assert.Empty(Directory.GetFiles(journalsDir, "*.journal"));
}
public void CanSyncWhenLazyTransactionsHasBufferedMultipleTransactions()
{
// we start by ensuring that we have enough space in the scratch buffers
PageFromScratchBuffer allocate;
using (var tx = Env.WriteTransaction())
{
allocate = Env.ScratchBufferPool.Allocate(tx.LowLevelTransaction, 60);
tx.Commit();
}
using (var tx = Env.WriteTransaction())
{
Env.ScratchBufferPool.Free(tx.LowLevelTransaction, allocate.ScratchFileNumber, allocate.PositionInScratchBuffer, tx.LowLevelTransaction.Id);
tx.Commit();
}
// then we run a lazy transaction
using (var tx = Env.WriteTransaction())
{
tx.LowLevelTransaction.IsLazyTransaction = true;
var tree = tx.CreateTree("foo");
tree.Add("test", "test");
tx.Commit();
}
// and another one, but if the pager isn't 4KB, the
// _journal_ is, and this leaves an empty 4KB between
// the entries
using (var tx = Env.WriteTransaction())
{
tx.LowLevelTransaction.IsLazyTransaction = true;
var tree = tx.CreateTree("foo");
tree.Add("test2", "test");
tx.Commit(); // this leaves 2 4kbs in the journal
}
// here we flush the lazy tx buffer
using (var tx = Env.WriteTransaction())
{
var tree = tx.CreateTree("foo");
tree.Add("test3", "test");
tx.Commit();
}
// and now we force them to sync
Env.FlushLogToDataFile();
using (var op = new WriteAheadJournal.JournalApplicator.SyncOperation(Env.Journal.Applicator))
{
op.SyncDataFile();
}
}
public void Recovery_must_not_delete_journals_that_havent_been_synced_yet()
{
RequireFileBasedPager();
var r = new Random();
var bytes = new byte[512];
for (int i = 0; i < 10; i++)
{
using (var tx = Env.WriteTransaction())
{
Tree tree = tx.CreateTree("tree");
for (int j = 0; j < 100; j++)
{
r.NextBytes(bytes);
tree.Add(new string((char)j, 1000), bytes);
}
tx.Commit();
}
}
Env.FlushLogToDataFile();
for (int i = 0; i < 10; i++)
{
using (var tx = Env.WriteTransaction())
{
Tree tree = tx.CreateTree("tree");
for (int j = 0; j < 100; j++)
{
r.NextBytes(bytes);
tree.Add(new string((char)j, 1000), bytes);
}
tx.Commit();
}
}
using (var operation = new WriteAheadJournal.JournalApplicator.SyncOperation(Env.Journal.Applicator)
{
AfterGatherInformationAction = () => Env.FlushLogToDataFile()
})
{
var syncResult = operation.SyncDataFile();
}
RestartDatabase();
}
public void Can_successfully_sync_journals_after_recovery()
{
RequireFileBasedPager();
var r = new Random(1);
var bytes = new byte[512];
for (int i = 0; i < 10; i++)
{
using (var tx = Env.WriteTransaction())
{
Tree tree = tx.CreateTree("tree");
for (int j = 0; j < 100; j++)
{
r.NextBytes(bytes);
tree.Add(new string((char)j, 1000), bytes);
}
tx.Commit();
}
}
Env.FlushLogToDataFile();
using (var operation = new WriteAheadJournal.JournalApplicator.SyncOperation(Env.Journal.Applicator))
{
operation.SyncDataFile();
}
for (int i = 0; i < 10; i++)
{
using (var tx = Env.WriteTransaction())
{
Tree tree = tx.CreateTree("tree");
for (int j = 0; j < 100; j++)
{
r.NextBytes(bytes);
tree.Add(new string((char)j, 1000), bytes);
}
tx.Commit();
}
}
Env.FlushLogToDataFile();
StopDatabase();
StartDatabase();
using (var operation = new WriteAheadJournal.JournalApplicator.SyncOperation(Env.Journal.Applicator))
{
operation.SyncDataFile();
}
StopDatabase();
StartDatabase();
using (var tx = Env.ReadTransaction())
{
using (var it = tx.ReadTree("tree").Iterate(prefetch: false))
{
Assert.True(it.Seek(Slices.BeforeAllKeys));
var count = 0;
do
{
count++;
} while (it.MoveNext());
Assert.Equal(100, count);
}
}
}
// As part of the sync operation, there are stages where the sync operation needs the flush lock
// and as part of the flush operation, there are stages the flush operation needs transaction write lock
// this can lead to a situation where the sync is waiting to flush waiting to write transaction
// so the sync pass his work that needs the flush lock to the flush operation if the lock is occupied
// and if the flush operation can do it while it waits to write transaction lock
//In this test, the sync is called while the flush is running and waiting to write transaction so the sync should not be blocked
public void CanSyncWhileFlushWaiteToWriteTransaction()
{
var syncMayFinishedEvent = new AutoResetEvent(false);
//Adding unsynced bytes so the sync thread will has work to do
for (var i = 0; i < 100; i++)
{
using (var tx = Env.WriteTransaction())
{
var tree = tx.CreateTree("foo");
tree.Add("items/" + i, StreamFor("values/" + i));
tx.Commit();
}
}
Env.FlushLogToDataFile();
//Adding unflushed bytes so the flush thread will has work to do
for (var i = 0; i < 100; i++)
{
using (var tx = Env.WriteTransaction())
{
var tree = tx.CreateTree("foo");
tree.Add("items/" + i, StreamFor("values/" + i));
tx.Commit();
}
}
void Sync()
{
try
{
using (var operation = new WriteAheadJournal.JournalApplicator.SyncOperation(Env.Journal.Applicator))
{
operation.SyncDataFile();
}
}
finally
{
syncMayFinishedEvent.Set();
}
}
void Flush()
{
try
{
using (Env.Journal.Applicator.TakeFlushingLock())
{
Task.Run((Action)Sync);
Env.FlushLogToDataFile();
}
}
catch (Exception)
{
syncMayFinishedEvent.Set();
}
}
// Write transaction lock is taken to block the flush
using (var tx = Env.WriteTransaction())
{
Task.Run((Action)Flush);
syncMayFinishedEvent.WaitOne(TimeSpan.FromSeconds(10));
var totalWrittenButUnsyncedBytes = Env.Journal.Applicator.TotalWrittenButUnsyncedBytes;
Assert.Equal(0, totalWrittenButUnsyncedBytes);
}
}
public void CorruptedSingleTransactionPage_WontStopTheRecoveryIfIgnoreErrorsOfSyncedTransactionIsSet()
{
RequireFileBasedPager();
using (var tx = Env.WriteTransaction())
{
tx.CreateTree("tree");
tx.Commit();
}
for (var i = 0; i < 100; i++)
{
var buffer = new byte[1000];
new Random().NextBytes(buffer);
using (var tx = Env.WriteTransaction())
{
for (int j = 0; j < 100; j++)
{
tx.CreateTree("tree").Add("a" + i.ToString() + j.ToString(), new MemoryStream(buffer));
}
tx.Commit();
}
}
var lastJournal = Env.Journal.GetCurrentJournalInfo().CurrentJournal;
// let's flush and sync
Env.FlushLogToDataFile();
using (var operation = new WriteAheadJournal.JournalApplicator.SyncOperation(Env.Journal.Applicator))
{
operation.SyncDataFile();
}
for (var i = 0; i < 100; i++)
{
var buffer = new byte[1000];
new Random().NextBytes(buffer);
using (var tx = Env.WriteTransaction())
{
for (int j = 0; j < 100; j++)
{
tx.CreateTree("tree").Add("b" + i.ToString() + j.ToString(), new MemoryStream(buffer));
}
tx.Commit();
}
}
StopDatabase();
CorruptJournal(lastJournal, 4 * Constants.Size.Kilobyte * 4);
StartDatabase();
Assert.True(_onIntegrityErrorOfAlreadySyncedDataHandlerWasCalled);
using (var tx = Env.ReadTransaction())
{
for (var i = 0; i < 100; i++)
{
for (int j = 0; j < 100; j++)
{
var readA = tx.ReadTree("tree").Read("a" + i.ToString() + j.ToString());
Assert.NotNull(readA);
var readB = tx.ReadTree("tree").Read("b" + i.ToString() + j.ToString());
Assert.NotNull(readB);
}
tx.Commit();
}
}
}
public void Full_backup_must_backup_journals_that_we_havent_synced_yet()
{
RequireFileBasedPager();
var r = new Random();
var bytes = new byte[512];
for (int i = 0; i < 10; i++)
{
using (var tx = Env.WriteTransaction())
{
Tree tree = tx.CreateTree("tree");
for (int j = 0; j < 100; j++)
{
r.NextBytes(bytes);
tree.Add(new string((char)j, 1000), bytes);
}
tx.Commit();
}
}
Env.FlushLogToDataFile();
for (int i = 0; i < 10; i++)
{
using (var tx = Env.WriteTransaction())
{
Tree tree = tx.CreateTree("tree");
for (int j = 0; j < 100; j++)
{
r.NextBytes(bytes);
tree.Add(new string((char)j, 1000), bytes);
}
tx.Commit();
}
}
using (var operation = new WriteAheadJournal.JournalApplicator.SyncOperation(Env.Journal.Applicator)
{
AfterGatherInformationAction = () =>
{
Env.FlushLogToDataFile();
}
})
{
var syncResult = operation.SyncDataFile();
}
var voronDataDir = new VoronPathSetting(DataDir);
BackupMethods.Full.ToFile(Env, voronDataDir.Combine("voron-test.backup"));
BackupMethods.Full.Restore(voronDataDir.Combine("voron-test.backup"), voronDataDir.Combine("backup-test.data"));
var options = StorageEnvironmentOptions.ForPath(Path.Combine(DataDir, "backup-test.data"));
options.MaxLogFileSize = Env.Options.MaxLogFileSize;
using (var env = new StorageEnvironment(options))
{
}
}
public void RaceConditionBetweenFullBackupAndUpdateDatabaseStateAfterSync()
{
RequireFileBasedPager();
var random = new Random(2);
var buffer = new byte[8192];
random.NextBytes(buffer);
using (var tx = Env.WriteTransaction())
{
var tree = tx.CreateTree("foo");
for (int i = 0; i < 5000; i++)
{
tree.Add("items/" + i, new MemoryStream(buffer));
}
tx.Commit();
}
Assert.True(Env.Journal.Files.Count > 1);
Env.FlushLogToDataFile(); // force writing data to the data file
var voronDataDir = new VoronPathSetting(DataDir);
Env.ForTestingPurposesOnly().ActionToCallDuringFullBackupRighAfterCopyHeaders += () =>
{
// here we remove 0000000000000000000.journal file while during backup we'll try to backup it
Thread syncOperation = new Thread(() =>
{
using (var operation = new WriteAheadJournal.JournalApplicator.SyncOperation(Env.Journal.Applicator))
{
var syncResult = operation.SyncDataFile();
Assert.True(syncResult);
}
});
syncOperation.Start();
Assert.False(syncOperation.Join(TimeSpan.FromSeconds(5)));
};
BackupMethods.Full.ToFile(Env, voronDataDir.Combine("voron-test.backup"));
BackupMethods.Full.Restore(voronDataDir.Combine("voron-test.backup"), voronDataDir.Combine("backup-test.data"));
var options = StorageEnvironmentOptions.ForPath(Path.Combine(DataDir, "backup-test.data"));
options.MaxLogFileSize = Env.Options.MaxLogFileSize;
using (var env = new StorageEnvironment(options))
{
using (var tx = env.ReadTransaction())
{
var tree = tx.CreateTree("foo");
for (int i = 0; i < 5000; i++)
{
var readResult = tree.Read("items/" + i);
Assert.NotNull(readResult);
var memoryStream = new MemoryStream();
readResult.Reader.CopyTo(memoryStream);
Assert.Equal(memoryStream.ToArray(), buffer);
}
}
}
}
public void MustNotReadAndProceedWithRecyclableButEffectivelyEmptyJournalOnRecovery()
{
RequireFileBasedPager();
long lastCommittedTxId = -1;
var r = new Random(10_09_2021);
for (int i = 0; i < 20; i++)
{
using (var tx = Env.WriteTransaction())
{
var tree = tx.CreateTree("foo");
tree.Add($"items/{i}", new byte[] { 1, 2, 3, (byte)i });
tx.Commit();
lastCommittedTxId = tx.LowLevelTransaction.Id;
}
}
Assert.Equal(2, Env.Journal.Files.Count);
Env.FlushLogToDataFile();
using (var operation = new WriteAheadJournal.JournalApplicator.SyncOperation(Env.Journal.Applicator))
{
operation.SyncDataFile();
}
Assert.Equal(1, Env.Journal.Files.Count);
var journalPath = ((StorageEnvironmentOptions.DirectoryStorageEnvironmentOptions)Env.Options).JournalPath.FullPath;
var journalsForReuse = new DirectoryInfo(journalPath).GetFiles($"{StorageEnvironmentOptions.RecyclableJournalFileNamePrefix}*");
Assert.Equal(1, journalsForReuse.Length);
using (var tx = Env.WriteTransaction())
{
// we are writing big values in this tx to ensure we'll have NextFile() call that will create a new journal (based on the recyclable journal file that exists)
for (int i = 0; i < 100; i++)
{
var bytes = new byte[2000];
r.NextBytes(bytes);
tx.CreateTree("bar").Add($"bigValues/{i}", bytes);
}
Assert.Throws <InvalidOperationException>(() =>
{
using (tx.LowLevelTransaction.ForTestingPurposesOnly().CallJustBeforeWritingToJournal(() => throw new InvalidOperationException()))
{
tx.Commit();
}
});
}
// we failed to commit the above transaction and write data to file but we managed to create _empty_ journal file
// this journal was created based on the recyclable journal which had old transactions already there that causes problems during recovery below
RestartDatabase();
using (var tx = Env.WriteTransaction())
{
Assert.Equal(lastCommittedTxId + 1, tx.LowLevelTransaction.Id);
}
}
public void TheCaseWithTwoEmptyJournalFiles()
{
RequireFileBasedPager();
long lastCommittedTxId = -1;
var r = new Random(10_09_2021);
for (int i = 0; i < 20; i++)
{
using (var tx = Env.WriteTransaction())
{
var tree = tx.CreateTree("foo");
tree.Add($"items/{i}", new byte[] { 1, 2, 3, (byte)i });
tx.Commit();
lastCommittedTxId = tx.LowLevelTransaction.Id;
}
}
Assert.Equal(2, Env.Journal.Files.Count);
var journalPath = ((StorageEnvironmentOptions.DirectoryStorageEnvironmentOptions)Env.Options).JournalPath.FullPath;
var journalsForReuse = new DirectoryInfo(journalPath).GetFiles($"{StorageEnvironmentOptions.RecyclableJournalFileNamePrefix}*");
Assert.Equal(0, journalsForReuse.Length);
using (var tx = Env.WriteTransaction())
{
// we are writing big values in this tx to ensure we'll have NextFile() call that will create a new journal (empty file - not recyclable)
for (int i = 0; i < 100; i++)
{
var bytes = new byte[2000];
r.NextBytes(bytes);
tx.CreateTree("bar").Add($"bigValues/{i}", bytes);
}
Assert.Throws <InvalidOperationException>(() =>
{
using (tx.LowLevelTransaction.ForTestingPurposesOnly().CallJustBeforeWritingToJournal(() => throw new InvalidOperationException()))
{
tx.Commit();
}
});
}
// we failed to commit the above transaction and write data to file but we managed to create _empty_ journal file
using (var operation = new WriteAheadJournal.JournalApplicator.SyncOperation(Env.Journal.Applicator))
{
operation.SyncDataFile();
}
RestartDatabase();
using (var tx = Env.WriteTransaction())
{
// we are writing big values in this tx to ensure we'll have NextFile() call that will create a new journal (again - empty file)
for (int i = 0; i < 100; i++)
{
var bytes = new byte[2000];
r.NextBytes(bytes);
tx.CreateTree("bar").Add($"bigValues/{i}", bytes);
}
Assert.Throws <InvalidOperationException>(() =>
{
using (tx.LowLevelTransaction.ForTestingPurposesOnly().CallJustBeforeWritingToJournal(() => throw new InvalidOperationException()))
{
tx.Commit();
}
});
}
// we failed to commit the above transaction and write data to file but we managed to create _empty_ journal file
// at this point we have 2 empty journal files and that caused Debug.Assert() to fail in WriteAheadJournal.RecoverDatabase()
using (var operation = new WriteAheadJournal.JournalApplicator.SyncOperation(Env.Journal.Applicator))
{
operation.SyncDataFile();
}
RestartDatabase();
using (var tx = Env.WriteTransaction())
{
Assert.Equal(lastCommittedTxId + 1, tx.LowLevelTransaction.Id);
}
}
public void Recovery_must_not_throw_missing_journal_if_we_have_synced_everything()
{
RequireFileBasedPager();
var r = new Random(1); // we must use seed to ensure we always fill up the journal
var bytes = new byte[512];
for (int i = 0; i < 10; i++)
{
using (var tx = Env.WriteTransaction())
{
Tree tree = tx.CreateTree("tree");
for (int j = 0; j < 100; j++)
{
r.NextBytes(bytes);
tree.Add(new string((char)j, 1000), bytes);
}
tx.Commit();
}
}
Env.FlushLogToDataFile();
for (int i = 0; i < 20; i++)
{
using (var tx = Env.WriteTransaction())
{
Tree tree = tx.CreateTree("tree");
for (int j = 0; j < 100; j++)
{
r.NextBytes(bytes);
tree.Add(new string((char)j, 1000), bytes);
}
tx.Commit();
}
}
// here we're filling up the last journal completely, we'll have Available4Kbs == 0 after the commit
using (var tx = Env.WriteTransaction())
{
Tree tree = tx.CreateTree("tree");
for (int j = 0; j < 226; j++)
{
var specialBytes = new byte[1024];
r.NextBytes(specialBytes);
tree.Add(new string((char)j, 1000), specialBytes);
}
tx.Commit();
}
Env.FlushLogToDataFile();
using (var operation = new WriteAheadJournal.JournalApplicator.SyncOperation(Env.Journal.Applicator))
{
var syncResult = operation.SyncDataFile();
}
RestartDatabase();
using (var tx = Env.WriteTransaction())
{
tx.CreateTree("foobar");
tx.Commit();
}
RestartDatabase();
using (var tx = Env.ReadTransaction())
{
using (var it = tx.ReadTree("tree").Iterate(prefetch: false))
{
Assert.True(it.Seek(Slices.BeforeAllKeys));
var count = 0;
do
{
count++;
} while (it.MoveNext());
Assert.Equal(226, count);
}
}
}
public void Recovery_should_handle_empty_journal_file_and_correctly_set_last_flushed_journal(bool runInMemory)
{
if (runInMemory == false)
{
RequireFileBasedPager();
}
using (var tx = Env.WriteTransaction())
{
tx.CreateTree("tree").Add("item", new byte[] { 1, 2, 3 });
tx.Commit();
}
var numberOfJournals = Env.Journal.Files.Count;
using (var tx = Env.WriteTransaction())
{
tx.CreateTree("tree").Add("item", new byte[] { 1, 2, 3 });
Assert.Throws <InvalidOperationException>(() =>
{
using (tx.LowLevelTransaction.ForTestingPurposesOnly().CallJustBeforeWritingToJournal(() => throw new InvalidOperationException()))
{
tx.Commit();
}
});
}
// we failed to commit the above transaction and write data to file
// but we managed to create _empty_ journal file
Assert.Equal(numberOfJournals + 1, Env.Journal.Files.Count);
RestartDatabase();
using (var operation = new WriteAheadJournal.JournalApplicator.SyncOperation(Env.Journal.Applicator))
{
// attempt to sync throws the following exception:
// System.InvalidOperationException : The lock task failed
// ----Voron.Exceptions.VoronUnrecoverableErrorException : Error syncing the data file.The last sync tx is 2, but the journal's last tx id is -1, possible file corruption?
operation.SyncDataFile();
}
// let's make sure we can restart immediately after sync
RestartDatabase();
// there is nothing to flush but let's validate it won't throw
using (var operation = new WriteAheadJournal.JournalApplicator.SyncOperation(Env.Journal.Applicator))
{
operation.SyncDataFile();
}
// let's make sure we can put more stuff there
for (int i = 0; i < 5; i++)
{
using (var tx = Env.WriteTransaction())
{
tx.CreateTree("tree").Add("item", new byte[] { 1, 2, 3 });
tx.Commit();
}
}
RestartDatabase();
Env.FlushLogToDataFile();
using (var operation = new WriteAheadJournal.JournalApplicator.SyncOperation(Env.Journal.Applicator))
{
operation.SyncDataFile();
}
// lets add more data once again and force flushing and syncing
for (int i = 0; i < 5; i++)
{
using (var tx = Env.WriteTransaction())
{
tx.CreateTree("tree").Add("item", new byte[] { 1, 2, 3 });
tx.Commit();
}
}
Env.FlushLogToDataFile();
using (var operation = new WriteAheadJournal.JournalApplicator.SyncOperation(Env.Journal.Applicator))
{
operation.SyncDataFile();
}
}
public void After_backup_and_restore_recovery_must_not_throw_missing_journal_if_we_have_synced_everything()
{
RequireFileBasedPager();
var r = new Random(1); // we must use seed to ensure we always fill up the journal
var bytes = new byte[512];
for (int i = 0; i < 10; i++)
{
using (var tx = Env.WriteTransaction())
{
Tree tree = tx.CreateTree("tree");
for (int j = 0; j < 100; j++)
{
r.NextBytes(bytes);
tree.Add(new string((char)j, 1000), bytes);
}
tx.Commit();
}
}
Env.FlushLogToDataFile();
for (int i = 0; i < 20; i++)
{
using (var tx = Env.WriteTransaction())
{
Tree tree = tx.CreateTree("tree");
for (int j = 0; j < 100; j++)
{
r.NextBytes(bytes);
tree.Add(new string((char)j, 1000), bytes);
}
tx.Commit();
}
}
// here we're filling up the last journal completely, we'll have Available4Kbs == 0 after the commit
using (var tx = Env.WriteTransaction())
{
Tree tree = tx.CreateTree("tree");
for (int j = 0; j < 226; j++)
{
var specialBytes = new byte[1024];
r.NextBytes(specialBytes);
tree.Add(new string((char)j, 1000), specialBytes);
}
tx.Commit();
}
Env.FlushLogToDataFile();
using (var operation = new WriteAheadJournal.JournalApplicator.SyncOperation(Env.Journal.Applicator))
{
var syncResult = operation.SyncDataFile();
}
var voronDataDir = new VoronPathSetting(DataDir);
BackupMethods.Full.ToFile(Env, voronDataDir.Combine("voron-test.backup"));
BackupMethods.Full.Restore(voronDataDir.Combine("voron-test.backup"), voronDataDir.Combine("backup-test.data"));
var options = StorageEnvironmentOptions.ForPath(Path.Combine(DataDir, "backup-test.data"));
options.MaxLogFileSize = Env.Options.MaxLogFileSize;
using (var env = new StorageEnvironment(options))
{
using (var tx = env.ReadTransaction())
{
using (var it = tx.ReadTree("tree").Iterate(prefetch: false))
{
Assert.True(it.Seek(Slices.BeforeAllKeys));
var count = 0;
do
{
count++;
} while (it.MoveNext());
Assert.Equal(226, count);
}
}
}
}
