public void CompactorShouldNotBePessimistDespiteRunningTransactions()
{
using var fileCollection = new InMemoryFileCollection();
var options = new KeyValueDBOptions
{
Compression = new NoCompressionStrategy(),
FileCollection = fileCollection,
FileSplitSize = 8096,
CompactorScheduler = CompactorScheduler.Instance,
};
using var kvDb = new KeyValueDB(options);
for (var i = 0; i < 100; i++)
{
using var tr = kvDb.StartWritingTransaction().Result;
var key = new byte[4];
BTDB.Buffer.PackUnpack.PackInt32BE(key, 0, i);
tr.CreateOrUpdateKeyValue(key, new byte[2000]);
tr.Commit();
}
kvDb.Compact(new CancellationToken());
var fileCountAfterFirstCompaction = fileCollection.GetCount();
using (kvDb.StartReadOnlyTransaction())
{
for (var i = 0; i < 50; i++)
{
using var tr = kvDb.StartWritingTransaction().Result;
var key = new byte[4];
BTDB.Buffer.PackUnpack.PackInt32BE(key, 0, i * 2);
tr.FindExactKey(key);
tr.EraseCurrent();
tr.Commit();
}
while (kvDb.Compact(new CancellationToken()))
{
}
Assert.InRange(fileCollection.GetCount(), fileCountAfterFirstCompaction + 2, fileCountAfterFirstCompaction + 50);
}
for (var i = 0; i < 4; i++)
{
using var tr = kvDb.StartWritingTransaction().Result;
var key = new byte[4];
BTDB.Buffer.PackUnpack.PackInt32BE(key, 0, i);
tr.CreateOrUpdateKeyValue(key, new byte[2000]);
tr.Commit();
}
while (kvDb.Compact(new CancellationToken()))
{
}
Assert.InRange(fileCollection.GetCount(), fileCountAfterFirstCompaction / 3, 2 * fileCountAfterFirstCompaction / 3);
}
void Create2TrlFiles(KeyValueDBOptions options)
{
using (var db = NewKeyValueDB(options))
{
using (var tr = db.StartTransaction())
{
tr.CreateOrUpdateKeyValue(_key1, new byte[1024]);
tr.Commit();
}
using (var tr = db.StartTransaction())
{
tr.CreateOrUpdateKeyValue(_key3, new byte[50]);
tr.Commit();
}
}
}
public void OpeningDbWithLenientOpenWithMissingFirstTrlAndKviWarnsAndRecoversData()
{
using var fileCollection = new InMemoryFileCollection();
var options = new KeyValueDBOptions
{
FileCollection = fileCollection, Compression = new NoCompressionStrategy(), FileSplitSize = 1024,
LenientOpen = true,
Logger = new LoggerMock()
};
Create2TrlFiles(options);
fileCollection.GetFile(1) !.Remove();
using var db = NewKeyValueDB(options);
Assert.Equal("No valid Kvi and lowest Trl in chain is not first. Missing 1. LenientOpen is true, recovering data.", ((LoggerMock)options.Logger).LastWarning);
using var tr = db.StartTransaction();
Assert.Equal(1, tr.GetKeyValueCount());
Assert.Equal(1u, fileCollection.GetCount());
}
public void CanOpenDbAfterDeletingAndCompacting()
{
using (var fileCollection = new InMemoryFileCollection())
{
var options = new KeyValueDBOptions
{
Compression = new NoCompressionStrategy(),
FileCollection = fileCollection,
FileSplitSize = 4096,
OpenUpToCommitUlong = null,
PreserveHistoryUpToCommitUlong = null,
CompactorScheduler = null,
};
using (var kvDb = new KeyValueDB(options))
{
using (var tr = kvDb.StartWritingTransaction().Result)
{
tr.CreateOrUpdateKeyValue(new byte[5], new byte[3000]);
tr.CreateOrUpdateKeyValue(new byte[6], new byte[2000]);
tr.Commit();
}
kvDb.Compact(CancellationToken.None);
}
using (var kvDb = new KeyValueDB(options))
{
using (var tr = kvDb.StartWritingTransaction().Result)
{
tr.FindFirstKey();
tr.EraseCurrent();
tr.Commit();
}
kvDb.Compact(CancellationToken.None);
}
using (var kvDb = new KeyValueDB(options))
{
// If there is error in KVI 3 it will create new KVI 4, but there is no problem in KVI 3
Assert.Null(kvDb.FileCollection.FileInfoByIdx(4));
}
}
}
public void OpeningDbWithMissingFirstTrlAndKviWarnsAndOpenEmptyDb()
{
using var fileCollection = new InMemoryFileCollection();
var options = new KeyValueDBOptions
{
FileCollection = fileCollection, Compression = new NoCompressionStrategy(), FileSplitSize = 1024,
Logger = new LoggerMock()
};
Create2TrlFiles(options);
fileCollection.GetFile(1) !.Remove();
using var db = NewKeyValueDB(options);
Assert.Equal("No valid Kvi and lowest Trl in chain is not first. Missing 1", ((LoggerMock)options.Logger).LastWarning);
using var tr = db.StartTransaction();
Assert.Equal(0, tr.GetKeyValueCount());
Assert.Equal(0u, fileCollection.GetCount());
tr.CreateOrUpdateKeyValue(_key1, new byte[1024]);
tr.Commit();
}
public void CompatorShouldNotBePesimist()
{
using (var fileCollection = new InMemoryFileCollection())
{
var options = new KeyValueDBOptions
{
Compression = new NoCompressionStrategy(),
FileCollection = fileCollection,
FileSplitSize = 8096,
OpenUpToCommitUlong = null,
PreserveHistoryUpToCommitUlong = null,
CompactorScheduler = CompactorScheduler.Instance,
};
using (var kvDb = new KeyValueDB(options))
{
for (var i = 0; i < 100; i++)
{
using (var tr = kvDb.StartWritingTransaction().Result)
{
var key = new byte[4];
BTDB.Buffer.PackUnpack.PackInt32BE(key, 0, i);
tr.CreateOrUpdateKeyValueUnsafe(key, new byte[2000]);
tr.SetCommitUlong((ulong)i);
tr.Commit();
}
}
kvDb.PreserveHistoryUpToCommitUlong = 100;
kvDb.Compact(new System.Threading.CancellationToken());
var fileCountAfterFirstCompaction = fileCollection.GetCount();
for (var i = 0; i < 50; i++)
{
using (var tr = kvDb.StartWritingTransaction().Result)
{
var key = new byte[4];
BTDB.Buffer.PackUnpack.PackInt32BE(key, 0, i);
tr.FindExactKey(key);
tr.EraseCurrent();
tr.SetCommitUlong(100 + (ulong)i);
tr.Commit();
}
}
kvDb.PreserveHistoryUpToCommitUlong = 150;
kvDb.Compact(new System.Threading.CancellationToken());
Assert.InRange(fileCollection.GetCount(), fileCountAfterFirstCompaction + 1, fileCountAfterFirstCompaction + 3);
using (var trOlder = kvDb.StartReadOnlyTransaction())
{
for (var i = 50; i < 100; i++)
{
using (var tr = kvDb.StartWritingTransaction().Result)
{
var key = new byte[4];
BTDB.Buffer.PackUnpack.PackInt32BE(key, 0, i);
tr.FindExactKey(key);
tr.EraseCurrent();
tr.SetCommitUlong(100 + (ulong)i);
tr.Commit();
}
}
kvDb.Compact(new System.Threading.CancellationToken());
Assert.InRange(fileCollection.GetCount(), fileCountAfterFirstCompaction / 3, 2 * fileCountAfterFirstCompaction / 3);
kvDb.PreserveHistoryUpToCommitUlong = 200;
kvDb.Compact(new System.Threading.CancellationToken());
Assert.InRange(fileCollection.GetCount(), fileCountAfterFirstCompaction / 3, 2 * fileCountAfterFirstCompaction / 3);
}
kvDb.Compact(new System.Threading.CancellationToken());
Assert.InRange <uint>(fileCollection.GetCount(), 1, 4);
}
}
}
public void ComplexTrlRollbackWhenKviLost()
{
using (var fileCollection = new InMemoryFileCollection())
{
var options = new KeyValueDBOptions
{
Compression = new SnappyCompressionStrategy(),
FileCollection = fileCollection,
FileSplitSize = 100 * 1024 * 1024,
OpenUpToCommitUlong = null,
PreserveHistoryUpToCommitUlong = null,
CompactorScheduler = CompactorScheduler.Instance,
};
using (var kvDb = new KeyValueDB(options))
using (var objDb = new ObjectDB())
{
objDb.Open(kvDb, false);
for (ulong i = 0; i < 100; i += 3)
{
using (var tr = objDb.StartWritingTransaction().Result)
{
var person = tr.Singleton <Person>();
person.Age = (uint)i;
tr.Store(person);
tr.SetCommitUlong(i);
tr.Commit();
}
}
}
options = new KeyValueDBOptions
{
Compression = new SnappyCompressionStrategy(),
FileCollection = fileCollection,
FileSplitSize = 100 * 1024 * 1024,
OpenUpToCommitUlong = 9UL,
PreserveHistoryUpToCommitUlong = 9UL,
CompactorScheduler = CompactorScheduler.Instance,
};
using (var kvDb = new KeyValueDB(options))
using (var objDb = new ObjectDB())
{
objDb.Open(kvDb, false);
using (var tr = objDb.StartReadOnlyTransaction())
{
Assert.Equal(9UL, tr.GetCommitUlong());
}
}
// Delete KVI file
fileCollection.GetFile(3).Remove();
options = new KeyValueDBOptions
{
Compression = new SnappyCompressionStrategy(),
FileCollection = fileCollection,
OpenUpToCommitUlong = 9UL,
PreserveHistoryUpToCommitUlong = 9UL,
FileSplitSize = 100 * 1024 * 1024,
CompactorScheduler = CompactorScheduler.Instance,
};
using (var kvDb = new KeyValueDB(options))
using (var objDb = new ObjectDB())
{
objDb.Open(kvDb, false);
using (var tr = objDb.StartReadOnlyTransaction())
{
Assert.Equal(9UL, tr.GetCommitUlong());
}
for (ulong i = 10; i < 200; i += 5)
{
using (var tr = objDb.StartWritingTransaction().Result)
{
var person = tr.Singleton <Person>();
person.Age = (uint)i;
tr.Store(person);
tr.SetCommitUlong(i);
tr.Commit();
}
}
}
options = new KeyValueDBOptions
{
Compression = new SnappyCompressionStrategy(),
FileCollection = fileCollection,
FileSplitSize = 100 * 1024 * 1024,
OpenUpToCommitUlong = 50UL,
PreserveHistoryUpToCommitUlong = 50UL,
CompactorScheduler = CompactorScheduler.Instance,
};
using (var kvDb = new KeyValueDB(options))
using (var objDb = new ObjectDB())
{
objDb.Open(kvDb, false);
using (var tr = objDb.StartReadOnlyTransaction())
{
Assert.Equal(50UL, tr.GetCommitUlong());
}
}
}
}
public override IKeyValueDB NewKeyValueDB(KeyValueDBOptions options)
{
return(new KeyValueDB(options));
}
public override IKeyValueDB NewKeyValueDB(KeyValueDBOptions options)
{
options.Allocator = _allocator;
return(new BTreeKeyValueDB(options));
}
protected abstract IKeyValueDB NewKeyValueDB(KeyValueDBOptions options);
protected override IKeyValueDB NewKeyValueDB(KeyValueDBOptions options)
{
return(new KeyValueDB(options));
}