public void CompactionStabilizedEvenWithOldTransactions()
{
using var fileCollection = new InMemoryFileCollection();
using var db = NewKeyValueDB(fileCollection, new NoCompressionStrategy(), 10240, null);
using (var tr = db.StartTransaction())
{
tr.CreateOrUpdateKeyValue(_key1, new byte[4000]);
tr.CreateOrUpdateKeyValue(Key2, new byte[4000]);
tr.Commit();
}
using (var tr = db.StartTransaction())
{
tr.CreateOrUpdateKeyValue(_key3, new byte[4000]); // creates new Log
tr.FindExactKey(_key1);
tr.EraseCurrent();
tr.Commit();
}
var longTr = db.StartTransaction();
db.Compact(new CancellationToken());
Assert.Equal(4u, fileCollection.GetCount()); // 2 Logs, 1 values, 1 KeyIndex
db.Compact(new CancellationToken());
Assert.Equal(4u, fileCollection.GetCount()); // 2 Logs, 1 values, 1 KeyIndex
longTr.Dispose();
db.Compact(new CancellationToken());
Assert.Equal(3u, fileCollection.GetCount()); // 1 Log, 1 values, 1 KeyIndex
}
public void FastCleanUpOnStartRemovesUselessFiles()
{
using (var fileCollection = new InMemoryFileCollection())
{
using (var db = new KeyValueDB(fileCollection, new NoCompressionStrategy(), 1024))
{
using (var tr = db.StartTransaction())
{
tr.CreateOrUpdateKeyValue(_key1, new byte[1024]);
tr.CreateOrUpdateKeyValue(Key2, new byte[1024]);
tr.Commit();
}
using (var tr = db.StartTransaction())
{
tr.EraseAll();
tr.Commit();
}
Assert.Equal(3u, fileCollection.GetCount()); // 3 Logs
}
using (var db = new KeyValueDB(fileCollection, new NoCompressionStrategy(), 1024))
{
Console.WriteLine(db.CalcStats());
Assert.Equal(2u, fileCollection.GetCount()); // 1 Log, 1 KeyIndex
}
}
}
public void AddingContinueToSameFileAfterReopenOfDBWith2TransactionLogFiles()
{
using (var fileCollection = new InMemoryFileCollection())
{
using (IKeyValueDB db = new KeyValueDB(fileCollection, new NoCompressionStrategy(), 1024))
{
using (var tr = db.StartTransaction())
{
tr.CreateOrUpdateKeyValue(_key1, new byte[1024]);
tr.CreateOrUpdateKeyValue(Key2, new byte[10]);
tr.Commit();
}
}
Assert.Equal(2u, fileCollection.GetCount());
using (IKeyValueDB db = new KeyValueDB(fileCollection, new NoCompressionStrategy(), 1024))
{
using (var tr = db.StartTransaction())
{
tr.CreateOrUpdateKeyValue(Key2, new byte[1024]);
tr.CreateOrUpdateKeyValue(_key3, new byte[10]);
tr.Commit();
}
}
Assert.Equal(4u, fileCollection.GetCount());
using (IKeyValueDB db = new KeyValueDB(fileCollection, new NoCompressionStrategy(), 1024))
{
using (var tr = db.StartTransaction())
{
tr.CreateOrUpdateKeyValue(Key2, Key2);
tr.Commit();
}
}
Assert.Equal(4u, fileCollection.GetCount());
}
}
public void CompactorShouldNotBePessimistDespiteRunningTransactions()
{
using var fileCollection = new InMemoryFileCollection();
var options = new KeyValueDBOptions
{
Compression = new NoCompressionStrategy(),
FileCollection = fileCollection,
FileSplitSize = 8096,
CompactorScheduler = CompactorScheduler.Instance,
};
using var kvDb = new KeyValueDB(options);
for (var i = 0; i < 100; i++)
{
using var tr = kvDb.StartWritingTransaction().Result;
var key = new byte[4];
BTDB.Buffer.PackUnpack.PackInt32BE(key, 0, i);
tr.CreateOrUpdateKeyValue(key, new byte[2000]);
tr.Commit();
}
kvDb.Compact(new CancellationToken());
var fileCountAfterFirstCompaction = fileCollection.GetCount();
using (kvDb.StartReadOnlyTransaction())
{
for (var i = 0; i < 50; i++)
{
using var tr = kvDb.StartWritingTransaction().Result;
var key = new byte[4];
BTDB.Buffer.PackUnpack.PackInt32BE(key, 0, i * 2);
tr.FindExactKey(key);
tr.EraseCurrent();
tr.Commit();
}
while (kvDb.Compact(new CancellationToken()))
{
}
Assert.InRange(fileCollection.GetCount(), fileCountAfterFirstCompaction + 2, fileCountAfterFirstCompaction + 50);
}
for (var i = 0; i < 4; i++)
{
using var tr = kvDb.StartWritingTransaction().Result;
var key = new byte[4];
BTDB.Buffer.PackUnpack.PackInt32BE(key, 0, i);
tr.CreateOrUpdateKeyValue(key, new byte[2000]);
tr.Commit();
}
while (kvDb.Compact(new CancellationToken()))
{
}
Assert.InRange(fileCollection.GetCount(), fileCountAfterFirstCompaction / 3, 2 * fileCountAfterFirstCompaction / 3);
}
public void AddingContinueToNewFileAfterReopenWithCorruption()
{
using (var fileCollection = new InMemoryFileCollection())
{
using (IKeyValueDB db = new KeyValueDB(fileCollection))
{
using (var tr = db.StartTransaction())
{
tr.CreateOrUpdateKeyValue(_key1, _key1);
tr.Commit();
}
}
fileCollection.SimulateCorruptionBySetSize(20 + 16);
using (IKeyValueDB db = new KeyValueDB(fileCollection))
{
using (var tr = db.StartTransaction())
{
Assert.Equal(0, tr.GetKeyValueCount());
tr.CreateOrUpdateKeyValue(Key2, Key2);
tr.Commit();
}
Console.WriteLine(db.CalcStats());
}
Assert.True(2 <= fileCollection.GetCount());
}
}
public void AddingContinueToSameFileAfterReopen()
{
using var fileCollection = new InMemoryFileCollection();
using (var db = NewKeyValueDB(fileCollection))
{
using (var tr = db.StartTransaction())
{
tr.CreateOrUpdateKeyValue(_key1, _key1);
tr.Commit();
}
}
using (var db = NewKeyValueDB(fileCollection))
{
using (var tr = db.StartTransaction())
{
tr.CreateOrUpdateKeyValue(Key2, Key2);
tr.Commit();
}
_testOutputHelper.WriteLine(db.CalcStats());
}
Assert.Equal(1u, fileCollection.GetCount()); // Log
}
public void CompactionWaitsForFinishingOldTransactionsBeforeRemovingFiles()
{
using (var fileCollection = new InMemoryFileCollection())
{
using (var db = new KeyValueDB(fileCollection, new NoCompressionStrategy(), 1024, null))
{
using (var tr = db.StartTransaction())
{
tr.CreateOrUpdateKeyValue(_key1, new byte[1024]);
tr.CreateOrUpdateKeyValue(Key2, new byte[10]);
tr.Commit();
}
var longTr = db.StartTransaction();
using (var tr = db.StartTransaction())
{
tr.FindExactKey(_key1);
tr.EraseCurrent();
tr.Commit();
}
Task.Run(() =>
{
db.Compact(new CancellationToken());
});
Thread.Sleep(2000);
Console.WriteLine(db.CalcStats());
Assert.True(4 <= fileCollection.GetCount()); // 2 Logs, 1 Value, 1 KeyIndex, (optinal 1 Unknown (old KeyIndex))
longTr.Dispose();
Thread.Sleep(1000);
Assert.Equal(2u, fileCollection.GetCount()); // 1 Log, 1 KeyIndex
using (var tr = db.StartTransaction())
{
tr.CreateOrUpdateKeyValue(_key3, new byte[10]);
tr.Commit();
}
using (var db2 = new KeyValueDB(fileCollection, new NoCompressionStrategy(), 1024))
{
using (var tr = db2.StartTransaction())
{
Assert.True(tr.FindExactKey(_key3));
}
}
}
}
}
public void CompactionDoesNotRemoveStillUsedFiles()
{
using (var fileCollection = new InMemoryFileCollection())
{
using (var db = new KeyValueDB(fileCollection, new NoCompressionStrategy(), 1024, null))
{
using (var tr = db.StartTransaction())
{
tr.CreateOrUpdateKeyValue(_key1, new byte[1024]);
tr.CreateOrUpdateKeyValue(Key2, new byte[10]);
tr.Commit();
}
var longTr = db.StartTransaction();
using (var tr = db.StartTransaction())
{
tr.FindExactKey(_key1);
tr.EraseCurrent();
tr.Commit();
}
db.Compact(new CancellationToken());
Assert.Equal(3u, fileCollection.GetCount()); // 2 Logs, 1 KeyIndex
longTr.Dispose();
db.Compact(new CancellationToken());
Assert.Equal(2u, fileCollection.GetCount()); // 1 Log, 1 KeyIndex
using (var tr = db.StartTransaction())
{
tr.CreateOrUpdateKeyValue(_key3, new byte[10]);
tr.Commit();
}
using (var db2 = new KeyValueDB(fileCollection, new NoCompressionStrategy(), 1024))
{
using (var tr = db2.StartTransaction())
{
Assert.True(tr.FindExactKey(_key3));
}
}
}
}
}
public void OpeningDbWithLenientOpenWithMissingFirstTrlAndKviWarnsAndRecoversData()
{
using var fileCollection = new InMemoryFileCollection();
var options = new KeyValueDBOptions
{
FileCollection = fileCollection, Compression = new NoCompressionStrategy(), FileSplitSize = 1024,
LenientOpen = true,
Logger = new LoggerMock()
};
Create2TrlFiles(options);
fileCollection.GetFile(1) !.Remove();
using var db = NewKeyValueDB(options);
Assert.Equal("No valid Kvi and lowest Trl in chain is not first. Missing 1. LenientOpen is true, recovering data.", ((LoggerMock)options.Logger).LastWarning);
using var tr = db.StartTransaction();
Assert.Equal(1, tr.GetKeyValueCount());
Assert.Equal(1u, fileCollection.GetCount());
}
public void OpeningDbWithMissingFirstTrlAndKviWarnsAndOpenEmptyDb()
{
using var fileCollection = new InMemoryFileCollection();
var options = new KeyValueDBOptions
{
FileCollection = fileCollection, Compression = new NoCompressionStrategy(), FileSplitSize = 1024,
Logger = new LoggerMock()
};
Create2TrlFiles(options);
fileCollection.GetFile(1) !.Remove();
using var db = NewKeyValueDB(options);
Assert.Equal("No valid Kvi and lowest Trl in chain is not first. Missing 1", ((LoggerMock)options.Logger).LastWarning);
using var tr = db.StartTransaction();
Assert.Equal(0, tr.GetKeyValueCount());
Assert.Equal(0u, fileCollection.GetCount());
tr.CreateOrUpdateKeyValue(_key1, new byte[1024]);
tr.Commit();
}
public void ReportTransactionLeak()
{
using var fileCollection = new InMemoryFileCollection();
using var db = NewKeyValueDB(fileCollection, new NoCompressionStrategy(), 1024);
var logger = new LoggerMock();
db.Logger = logger;
using (var tr = db.StartTransaction())
{
tr.CreateOrUpdateKeyValue(_key1, new byte[1]);
tr.Commit();
}
Assert.Equal(fileCollection.GetCount(), logger.TrlCreatedCount);
StartLeakingTransaction(db);
GC.Collect(GC.MaxGeneration);
GC.WaitForPendingFinalizers();
Assert.NotNull(logger.Leaked);
Assert.Equal("Leak", logger.Leaked.DescriptionForLeaks);
}
public void AddingContinueToSameFileAfterReopen()
{
using (var fileCollection = new InMemoryFileCollection())
{
using (IKeyValueDB db = new KeyValueDB(fileCollection))
{
using (var tr = db.StartTransaction())
{
tr.CreateOrUpdateKeyValue(_key1, _key1);
tr.Commit();
}
}
using (IKeyValueDB db = new KeyValueDB(fileCollection))
{
using (var tr = db.StartTransaction())
{
tr.CreateOrUpdateKeyValue(Key2, Key2);
tr.Commit();
}
Console.WriteLine(db.CalcStats());
}
Assert.Equal(2u, fileCollection.GetCount()); // Log + Index
}
}
public void CompatorShouldNotBePesimist()
{
using (var fileCollection = new InMemoryFileCollection())
{
var options = new KeyValueDBOptions
{
Compression = new NoCompressionStrategy(),
FileCollection = fileCollection,
FileSplitSize = 8096,
OpenUpToCommitUlong = null,
PreserveHistoryUpToCommitUlong = null,
CompactorScheduler = CompactorScheduler.Instance,
};
using (var kvDb = new KeyValueDB(options))
{
for (var i = 0; i < 100; i++)
{
using (var tr = kvDb.StartWritingTransaction().Result)
{
var key = new byte[4];
BTDB.Buffer.PackUnpack.PackInt32BE(key, 0, i);
tr.CreateOrUpdateKeyValueUnsafe(key, new byte[2000]);
tr.SetCommitUlong((ulong)i);
tr.Commit();
}
}
kvDb.PreserveHistoryUpToCommitUlong = 100;
kvDb.Compact(new System.Threading.CancellationToken());
var fileCountAfterFirstCompaction = fileCollection.GetCount();
for (var i = 0; i < 50; i++)
{
using (var tr = kvDb.StartWritingTransaction().Result)
{
var key = new byte[4];
BTDB.Buffer.PackUnpack.PackInt32BE(key, 0, i);
tr.FindExactKey(key);
tr.EraseCurrent();
tr.SetCommitUlong(100 + (ulong)i);
tr.Commit();
}
}
kvDb.PreserveHistoryUpToCommitUlong = 150;
kvDb.Compact(new System.Threading.CancellationToken());
Assert.InRange(fileCollection.GetCount(), fileCountAfterFirstCompaction + 1, fileCountAfterFirstCompaction + 3);
using (var trOlder = kvDb.StartReadOnlyTransaction())
{
for (var i = 50; i < 100; i++)
{
using (var tr = kvDb.StartWritingTransaction().Result)
{
var key = new byte[4];
BTDB.Buffer.PackUnpack.PackInt32BE(key, 0, i);
tr.FindExactKey(key);
tr.EraseCurrent();
tr.SetCommitUlong(100 + (ulong)i);
tr.Commit();
}
}
kvDb.Compact(new System.Threading.CancellationToken());
Assert.InRange(fileCollection.GetCount(), fileCountAfterFirstCompaction / 3, 2 * fileCountAfterFirstCompaction / 3);
kvDb.PreserveHistoryUpToCommitUlong = 200;
kvDb.Compact(new System.Threading.CancellationToken());
Assert.InRange(fileCollection.GetCount(), fileCountAfterFirstCompaction / 3, 2 * fileCountAfterFirstCompaction / 3);
}
kvDb.Compact(new System.Threading.CancellationToken());
Assert.InRange <uint>(fileCollection.GetCount(), 1, 4);
}
}
}
public void Run()
{
var collection = new InMemoryFileCollection();
using (var kvDb = new KeyValueDB(collection, new NoCompressionStrategy(), 100 * 1024 * 1024))
{
ulong itemsCount = 0;
using (var objDb = new ObjectDB())
{
objDb.Open(kvDb, false);
Console.WriteLine("started generating");
using (var tr = objDb.StartWritingTransaction().Result)
{
var objects = tr.Singleton <SmallObjects>();
while (true)
{
objects.Items.Add(itemsCount, new SmallObject()
{
Id = itemsCount, Label = "bu"
});
if (itemsCount % 1_000_000 == 0)
{
Console.WriteLine("Generated {0}", itemsCount);
}
if (itemsCount % 1000 == 0 && collection.GetCount() == 20)
{
break;
}
itemsCount++;
}
tr.Commit();
}
Console.WriteLine("finished generating");
using (var tr = objDb.StartWritingTransaction().Result)
{
var objects = tr.Singleton <SmallObjects>();
itemsCount = (ulong)objects.Items.Count;
tr.Commit();
}
Console.WriteLine("removing items started");
using (var tr = objDb.StartWritingTransaction().Result)
{
var objects = tr.Singleton <SmallObjects>();
for (ulong i = 0; i < itemsCount / 5; i++)
{
if (i % 2 == 0)
{
continue;
}
objects.Items.Remove(i);
}
tr.Commit();
}
Console.WriteLine("removing items finished");
var transactionCreationStarted = new ManualResetEventSlim(false);
var compactionFinished = new ManualResetEventSlim(false);
Task.Run(() =>
{
Console.WriteLine("Started waiting for transaction creating");
transactionCreationStarted.Wait();
Console.WriteLine("Started Compacting");
Trace.Assert(kvDb.Compact(CancellationToken.None));
Console.WriteLine("Finished Compacting");
compactionFinished.Set();
});
Console.WriteLine("Started concurrent transaction creation");
long msMax = 0;
long average = 0;
long iterations = 0;
Stopwatch watch = new Stopwatch();
while (true)
{
var compactionFinishedBeforeLasttransaction = compactionFinished.IsSet;
iterations++;
watch.Start();
var task = objDb.StartWritingTransaction();
if (!transactionCreationStarted.IsSet)
{
transactionCreationStarted.Set();
}
task.Wait();
var ms = watch.ElapsedMilliseconds;
average += ms;
msMax = Math.Max(ms, msMax);
watch.Reset();
using (var tr = task.Result)
{
tr.Commit();
}
if ((compactionFinishedBeforeLasttransaction && compactionFinished.IsSet))
{
break;
}
}
Console.WriteLine("Finished concurrent transaction creation, longest transaction create time was {0}ms, " +
"average {1}ms, iterations {2}", msMax, average / (double)iterations, iterations);
}
}
}