static string DumpRelationContent(string dbDir, string relationName, ulong?openUpToCommitUlong)
{
using var dfc = new OnDiskFileCollection(dbDir);
using var kdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = dfc,
ReadOnly = true,
OpenUpToCommitUlong = openUpToCommitUlong
});
using var odb = new ObjectDB();
odb.Open(kdb, false);
using var trkv = kdb.StartReadOnlyTransaction();
using var tr = odb.StartTransaction();
var visitor = new ToStringFastVisitor();
var iterator = new ODBIterator(tr, visitor);
iterator.LoadGlobalInfo();
var relationIdInfo = iterator.RelationId2Info.FirstOrDefault(kvp => kvp.Value.Name.EndsWith(relationName));
if (relationIdInfo.Value == null)
{
return("");
}
iterator.IterateRelation(relationIdInfo.Value);
return(visitor.ToString());
}
static ObjectDB CreateDb(IFileCollection fc)
{
var lowDb = new KeyValueDB(fc);
var db = new ObjectDB();
db.Open(lowDb, true);
return db;
}
public void AddingContinueToSameFileAfterReopenOfDBWith2TransactionLogFiles()
{
using (var fileCollection = new InMemoryFileCollection())
{
using (IKeyValueDB db = new KeyValueDB(fileCollection, new NoCompressionStrategy(), 1024))
{
using (var tr = db.StartTransaction())
{
tr.CreateOrUpdateKeyValue(_key1, new byte[1024]);
tr.CreateOrUpdateKeyValue(Key2, new byte[10]);
tr.Commit();
}
}
Assert.Equal(2u, fileCollection.GetCount());
using (IKeyValueDB db = new KeyValueDB(fileCollection, new NoCompressionStrategy(), 1024))
{
using (var tr = db.StartTransaction())
{
tr.CreateOrUpdateKeyValue(Key2, new byte[1024]);
tr.CreateOrUpdateKeyValue(_key3, new byte[10]);
tr.Commit();
}
}
Assert.Equal(4u, fileCollection.GetCount());
using (IKeyValueDB db = new KeyValueDB(fileCollection, new NoCompressionStrategy(), 1024))
{
using (var tr = db.StartTransaction())
{
tr.CreateOrUpdateKeyValue(Key2, Key2);
tr.Commit();
}
}
Assert.Equal(4u, fileCollection.GetCount());
}
}
public void RandomlyCreatedTest2()
{
using (var db = new KeyValueDB())
{
db.Open(CreateTestStream(), true);
using (var tr1 = db.StartTransaction())
{
tr1.CreateKey(new byte[] { 1 });
tr1.Commit();
}
using (var tr2 = db.StartTransaction())
{
tr2.CreateKey(new byte[] { 2 });
tr2.Commit();
}
using (var tr3 = db.StartTransaction())
{
tr3.CreateKey(new byte[] { 3 });
}
using (var tr4 = db.StartTransaction())
{
tr4.CreateKey(new byte[] { 4 });
}
}
}
public void BiggerKey([Values(0, 1, 268, 269, 270, 4364, 4365, 4366, 1200000)] int keyLength)
{
var key = new byte[keyLength];
for (int i = 0; i < keyLength; i++)
{
key[i] = (byte)i;
}
using (var stream = CreateTestStream())
{
using (IKeyValueDB db = new KeyValueDB())
{
db.Open(stream, false);
using (var tr1 = db.StartTransaction())
{
tr1.CreateKey(key);
tr1.Commit();
}
using (var tr2 = db.StartTransaction())
{
Assert.True(tr2.FindExactKey(key));
var buf = tr2.ReadKey();
Assert.AreEqual(key, buf);
Debug.WriteLine(tr2.CalculateStats().ToString());
}
}
Debug.WriteLine("KeySize:{0,7} DataBaseSize:{1,7}", keyLength, stream.GetSize());
}
}
public void TwoTransactions()
{
using (var fileCollection = new InMemoryFileCollection())
using (IKeyValueDB db = new KeyValueDB(fileCollection))
{
using (var tr1 = db.StartTransaction())
{
tr1.CreateKey(_key1);
tr1.Commit();
}
using (var tr2 = db.StartTransaction())
{
tr2.CreateKey(Key2);
Assert.True(tr2.FindExactKey(_key1));
Assert.True(tr2.FindExactKey(Key2));
Assert.False(tr2.FindExactKey(_key3));
tr2.Commit();
}
using (var tr3 = db.StartTransaction())
{
Assert.True(tr3.FindExactKey(_key1));
Assert.True(tr3.FindExactKey(Key2));
Assert.False(tr3.FindExactKey(_key3));
}
}
}
public void AddingContinueToNewFileAfterReopenWithCorruption()
{
using (var fileCollection = new InMemoryFileCollection())
{
using (IKeyValueDB db = new KeyValueDB(fileCollection))
{
using (var tr = db.StartTransaction())
{
tr.CreateOrUpdateKeyValue(_key1, _key1);
tr.Commit();
}
}
fileCollection.SimulateCorruptionBySetSize(20 + 16);
using (IKeyValueDB db = new KeyValueDB(fileCollection))
{
using (var tr = db.StartTransaction())
{
Assert.Equal(0, tr.GetKeyValueCount());
tr.CreateOrUpdateKeyValue(Key2, Key2);
tr.Commit();
}
Console.WriteLine(db.CalcStats());
}
Assert.True(2 <= fileCollection.GetCount());
}
}
public void SetKeyIndexWorks()
{
using (var fileCollection = new InMemoryFileCollection())
using (IKeyValueDB db = new KeyValueDB(fileCollection))
{
var key = new byte[2];
const int keysCreated = 10000;
using (var tr = db.StartTransaction())
{
for (int i = 0; i < keysCreated; i++)
{
key[0] = (byte)(i / 256);
key[1] = (byte)(i % 256);
tr.CreateKey(key);
}
tr.Commit();
}
using (var tr = db.StartTransaction())
{
Assert.False(tr.SetKeyIndex(keysCreated));
for (int i = 0; i < keysCreated; i += 5)
{
Assert.True(tr.SetKeyIndex(i));
key = tr.GetKeyAsByteArray();
Assert.Equal((byte)(i / 256), key[0]);
Assert.Equal((byte)(i % 256), key[1]);
Assert.Equal(i, tr.GetKeyIndex());
}
}
}
}
public void AddingContinueToNewFileAfterReopenWithCorruption()
{
using (var fileCollection = new InMemoryFileCollection())
{
using (IKeyValueDB db = new KeyValueDB(fileCollection))
{
using (var tr = db.StartTransaction())
{
tr.CreateOrUpdateKeyValue(_key1, _key1);
tr.Commit();
}
}
fileCollection.SimulateCorruptionBySetSize(20 + 16);
using (IKeyValueDB db = new KeyValueDB(fileCollection))
{
using (var tr = db.StartTransaction())
{
Assert.Equal(0, tr.GetKeyValueCount());
tr.CreateOrUpdateKeyValue(Key2, Key2);
tr.Commit();
}
Console.WriteLine(db.CalcStats());
}
Assert.True(2 <= fileCollection.GetCount());
}
}
public void MultipleTransactions2([Values(1000)] int transactionCount)
{
using (var stream = CreateTestStream())
using (IKeyValueDB db = new KeyValueDB())
{
db.Open(stream, false);
var key = new byte[2];
for (int i = 0; i < transactionCount; i++)
{
key[0] = (byte)((transactionCount - i) / 256);
key[1] = (byte)((transactionCount - i) % 256);
using (var tr1 = db.StartTransaction())
{
tr1.CreateKey(key);
if (i % 100 == 0 || i == transactionCount - 1)
{
for (int j = 0; j < i; j++)
{
key[0] = (byte)((transactionCount - j) / 256);
key[1] = (byte)((transactionCount - j) % 256);
Assert.True(tr1.FindExactKey(key));
}
}
tr1.Commit();
}
}
}
}
public void TwoTransactions()
{
using (var stream = CreateTestStream())
using (IKeyValueDB db = new KeyValueDB())
{
db.Open(stream, false);
using (var tr1 = db.StartTransaction())
{
tr1.CreateKey(_key1);
tr1.Commit();
}
using (var tr2 = db.StartTransaction())
{
tr2.CreateKey(_key2);
Assert.True(tr2.FindExactKey(_key1));
Assert.True(tr2.FindExactKey(_key2));
Assert.False(tr2.FindExactKey(_key3));
tr2.Commit();
}
using (var tr3 = db.StartTransaction())
{
Assert.True(tr3.FindExactKey(_key1));
Assert.True(tr3.FindExactKey(_key2));
Assert.False(tr3.FindExactKey(_key3));
}
}
}
void DoWork5ReadCheck()
{
_sw.Restart();
using (var stream = new PositionLessStreamProxy("data.btdb"))
using (IKeyValueDB db = new KeyValueDB())
{
db.Open(stream, false);
using (var trStat = db.StartTransaction())
{
Console.WriteLine(trStat.CalculateStats().ToString());
}
using (var trCheck = db.StartTransaction())
{
long pureDataLengthCheck = 0;
if (trCheck.FindFirstKey())
{
do
{
pureDataLengthCheck += trCheck.ReadKey().Length + trCheck.ReadValue().Length;
} while (trCheck.FindNextKey());
}
if (pureDataLengthCheck != 396130000)
{
throw new Exception("Bug");
}
}
using (var trStat = db.StartTransaction())
{
Console.WriteLine(trStat.CalculateStats().ToString());
}
}
_sw.Stop();
Console.WriteLine("Time: {0,15}ms", _sw.Elapsed.TotalMilliseconds);
}
public void SetValueWorksSameTransaction([Values(0, 1, 256, 268, 269, 270, 512, 4364, 4365, 4366, 5000, 1200000, 1200012, 10000000)] int firstLength,
[Values(0, 1, 256, 268, 269, 270, 512, 4364, 4365, 4366, 5000, 1200000, 1200012, 10000000)] int secondLength)
{
var valbuf = new byte[secondLength];
new Random(0).NextBytes(valbuf);
using (var stream = CreateTestStream())
using (IKeyValueDB db = new KeyValueDB())
{
db.Open(stream, false);
using (var tr1 = db.StartTransaction())
{
tr1.CreateKey(_key1);
tr1.CreateKey(_key2);
tr1.CreateKey(_key3);
tr1.SetValueSize(firstLength);
tr1.SetValue(valbuf, 0, valbuf.Length);
tr1.Commit();
}
using (var tr2 = db.StartTransaction())
{
Assert.True(tr2.FindExactKey(_key1));
Assert.True(tr2.FindExactKey(_key2));
Assert.True(tr2.FindExactKey(_key3));
var valbuf2 = tr2.ReadValue();
for (int i = 0; i < secondLength; i++)
{
if (valbuf[i] != valbuf2[i])
{
Assert.AreEqual(valbuf[i], valbuf2[i]);
}
}
}
}
}
void DoWork4()
{
var key = new byte[4000];
using (var stream = CreateTestStream())
using (IKeyValueDB db = new KeyValueDB())
{
db.Open(stream, false);
_sw.Restart();
for (int i = 0; i < 30000; i++)
{
using (var tr = db.StartTransaction())
{
key[3] = (byte)(i % 256);
key[2] = (byte)(i / 256 % 256);
key[1] = (byte)(i / 256 / 256 % 256);
key[0] = (byte)(i / 256 / 256 / 256);
tr.CreateOrUpdateKeyValue(key, key);
tr.Commit();
}
}
_sw.Stop();
using (var trStat = db.StartTransaction())
{
Console.WriteLine(trStat.CalculateStats().ToString());
Console.WriteLine("Insert COU: {0,15}ms", _sw.Elapsed.TotalMilliseconds);
}
}
}
public void PreapprovedCommitAndCompaction()
{
using (var fileCollection = new InMemoryFileCollection())
{
using (var db = new KeyValueDB(fileCollection, new NoCompressionStrategy(), 1024))
{
using (var tr = db.StartWritingTransaction().Result)
{
tr.CreateOrUpdateKeyValue(_key1, new byte[1024]);
tr.CreateOrUpdateKeyValue(Key2, new byte[10]);
tr.Commit();
}
db.Compact(new CancellationToken());
using (var tr = db.StartWritingTransaction().Result)
{
tr.EraseRange(0, 0);
tr.Commit();
}
db.Compact(new CancellationToken());
using (var db2 = new KeyValueDB(fileCollection, new NoCompressionStrategy(), 1024))
{
using (var tr = db2.StartTransaction())
{
Assert.False(tr.FindExactKey(_key1));
Assert.True(tr.FindExactKey(Key2));
}
}
}
}
}
public void ValueStoreWorksSameTransaction([Values(0, 1, 256, 268, 269, 270, 512, 4364, 4365, 4366, 5000, 1200000, 1200012, 10000000)] int firstLength,
[Values(0, 1, 256, 268, 269, 270, 512, 4364, 4365, 4366, 5000, 1200000, 1200012, 10000000)] int secondLength)
{
var valbuf = new byte[firstLength];
new Random(0).NextBytes(valbuf);
using (var stream = CreateTestStream())
using (IKeyValueDB db = new KeyValueDB())
{
db.Open(stream, false);
using (var tr1 = db.StartTransaction())
{
tr1.CreateKey(_key1);
tr1.WriteValue(0, valbuf.Length, valbuf, 0);
tr1.SetValueSize(secondLength);
var valbuf2 = tr1.ReadValue();
var commonLength = Math.Min(firstLength, secondLength);
for (int i = 0; i < commonLength; i++)
{
if (valbuf[i] != valbuf2[i])
{
Assert.AreEqual(valbuf[i], valbuf2[i]);
}
}
for (int i = commonLength; i < secondLength; i++)
{
if (valbuf2[i] != 0)
{
Assert.AreEqual(0, valbuf2[i]);
}
}
tr1.Commit();
}
}
}
public void CanRollbackToStartIfNoTrlMissing()
{
using (var fileCollection = new InMemoryFileCollection())
{
using (var kv = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = fileCollection,
FileSplitSize = 1024,
Compression = new NoCompressionStrategy()
}))
{
for (var i = 1; i < 60; i++)
{
using (var tr = kv.StartTransaction())
{
var key = new byte[4];
BTDB.Buffer.PackUnpack.PackInt32BE(key, 0, i);
tr.CreateOrUpdateKeyValueUnsafe(key, new byte[200]);
tr.SetCommitUlong((ulong)i);
tr.Commit();
}
if (i % 5 == 0)
{
kv.Compact(new System.Threading.CancellationToken());
}
if (i == 50)
{
kv.PreserveHistoryUpToCommitUlong = (ulong)i;
}
}
}
using (var kv = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = fileCollection,
FileSplitSize = 1024,
OpenUpToCommitUlong = 0,
Compression = new NoCompressionStrategy()
}))
{
using (var tr = kv.StartTransaction())
{
Assert.Equal(0, tr.GetKeyValueCount());
}
}
// Again after open with OpenUpToCommitUlong you lost option to replay old history
using (var kv = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = fileCollection,
FileSplitSize = 1024,
OpenUpToCommitUlong = 1,
Compression = new NoCompressionStrategy()
}))
{
using (var tr = kv.StartTransaction())
{
Assert.Equal(0, tr.GetKeyValueCount());
}
}
}
}
public void MultipleTransactions2(int transactionCount)
{
using (var fileCollection = new InMemoryFileCollection())
using (IKeyValueDB db = new KeyValueDB(fileCollection))
{
var key = new byte[2 + transactionCount * 10];
for (int i = 0; i < transactionCount; i++)
{
key[0] = (byte)((transactionCount - i) / 256);
key[1] = (byte)((transactionCount - i) % 256);
using (var tr1 = db.StartTransaction())
{
tr1.CreateOrUpdateKeyValue(ByteBuffer.NewSync(key, 0, 2 + i * 10), ByteBuffer.NewEmpty());
if (i % 100 == 0 || i == transactionCount - 1)
{
for (int j = 0; j < i; j++)
{
key[0] = (byte)((transactionCount - j) / 256);
key[1] = (byte)((transactionCount - j) % 256);
Assert.Equal(FindResult.Exact, tr1.Find(ByteBuffer.NewSync(key, 0, 2 + j * 10)));
}
}
tr1.Commit();
}
}
}
}
public void SetKeyPrefixInOneTransaction()
{
using (var fileCollection = new InMemoryFileCollection())
using (IKeyValueDB db = new KeyValueDB(fileCollection))
{
var key = new byte[5];
var value = new byte[100];
var rnd = new Random();
using (var tr = db.StartTransaction())
{
for (byte i = 0; i < 100; i++)
{
key[0] = i;
for (byte j = 0; j < 100; j++)
{
key[4] = j;
rnd.NextBytes(value);
tr.CreateOrUpdateKeyValue(key, value);
}
}
tr.Commit();
}
using (var tr = db.StartTransaction())
{
for (byte i = 0; i < 100; i++)
{
key[0] = i;
tr.SetKeyPrefix(ByteBuffer.NewSync(key, 0, 4));
Assert.Equal(100, tr.GetKeyValueCount());
}
}
}
}
public void FastCleanUpOnStartRemovesUselessFiles()
{
using (var fileCollection = new InMemoryFileCollection())
{
using (var db = new KeyValueDB(fileCollection, new NoCompressionStrategy(), 1024))
{
using (var tr = db.StartTransaction())
{
tr.CreateOrUpdateKeyValue(_key1, new byte[1024]);
tr.CreateOrUpdateKeyValue(Key2, new byte[1024]);
tr.Commit();
}
using (var tr = db.StartTransaction())
{
tr.EraseAll();
tr.Commit();
}
Assert.Equal(3u, fileCollection.GetCount()); // 3 Logs
}
using (var db = new KeyValueDB(fileCollection, new NoCompressionStrategy(), 1024))
{
Console.WriteLine(db.CalcStats());
Assert.Equal(2u, fileCollection.GetCount()); // 1 Log, 1 KeyIndex
}
}
}
public void StartWritingTransactionWorks()
{
using (var fileCollection = new InMemoryFileCollection())
using (IKeyValueDB db = new KeyValueDB(fileCollection))
{
var tr1 = db.StartWritingTransaction().Result;
var tr2Task = db.StartWritingTransaction();
var task = Task.Factory.StartNew(() =>
{
var tr2 = tr2Task.Result;
Assert.True(tr2.FindExactKey(_key1));
tr2.CreateKey(Key2);
tr2.Commit();
tr2.Dispose();
});
tr1.CreateKey(_key1);
tr1.Commit();
tr1.Dispose();
task.Wait(1000);
using (var tr = db.StartTransaction())
{
Assert.True(tr.FindExactKey(_key1));
Assert.True(tr.FindExactKey(Key2));
}
}
}
public void StartWritingTransactionWorks()
{
using (var stream = CreateTestStream())
using (IKeyValueDB db = new KeyValueDB())
{
db.Open(stream, false);
var tr1 = db.StartWritingTransaction().Result;
var tr2Task = db.StartWritingTransaction();
var task = Task.Factory.StartNew(() =>
{
var tr2 = tr2Task.Result;
Assert.True(tr2.FindExactKey(_key1));
tr2.CreateKey(_key2);
tr2.Commit();
tr2.Dispose();
});
tr1.CreateKey(_key1);
tr1.Commit();
tr1.Dispose();
task.Wait(1000);
using (var tr = db.StartTransaction())
{
Assert.True(tr.FindExactKey(_key1));
Assert.True(tr.FindExactKey(_key2));
}
}
}
public BenchmarkRelationPartialView()
{
_fc = new InMemoryFileCollection();
var lowDb = new KeyValueDB(_fc);
_db = new ObjectDB();
_db.Open(lowDb, true);
using var tr = _db.StartTransaction();
var table = tr.GetRelation <IPersonTable>();
for (var i = 0; i < 10000; i++)
{
var p = new Person
{
ParentId = 1,
PersonId = i,
Age = (ulong)(i / 128),
Name = "Lorem ipsum " + i,
Children = Enumerable.Range(0, 100).Select(j => new Person {
ParentId = i, PersonId = i * 100 + j, Name = "Lorem ipsum child " + j, Age = (ulong)j
}).ToList()
};
table.Upsert(p);
}
tr.Commit();
_tr = _db.StartReadOnlyTransaction();
_table = _tr.GetRelation <IPersonTable>();
}
public void BiggerKey(int prefixLength, int offsetKey, int keyLength)
{
var prefix = new byte[prefixLength];
var keyb = new byte[offsetKey + keyLength];
for (int i = offsetKey; i < offsetKey + keyLength; i++)
{
keyb[i] = (byte)i;
}
var key = ByteBuffer.NewAsync(keyb, offsetKey, keyLength);
using (var fileCollection = new InMemoryFileCollection())
using (IKeyValueDB db = new KeyValueDB(fileCollection))
{
using (var tr1 = db.StartTransaction())
{
tr1.SetKeyPrefix(prefix);
tr1.CreateOrUpdateKeyValue(key, ByteBuffer.NewEmpty());
tr1.Commit();
}
using (var tr2 = db.StartTransaction())
{
tr2.SetKeyPrefix(prefix);
Assert.True(tr2.FindExactKey(key.ToByteArray()));
Assert.Equal(key.ToByteArray(), tr2.GetKeyAsByteArray());
}
}
}
internal static int CalcKeyLenInline(int keyLen)
{
if (keyLen <= MaxKeyLenInline)
{
return(keyLen);
}
return(keyLen - KeyValueDB.RoundToAllocationGranularity(keyLen - MaxKeyLenInline));
}
static ObjectDB CreateDb(IFileCollection fc)
{
var lowDb = new KeyValueDB(fc);
var db = new ObjectDB();
db.Open(lowDb, true);
return(db);
}
internal static int CalcValueLenInline(long valueLen)
{
if (valueLen <= MaxValueLenInline)
{
return((int)valueLen);
}
return((int)(valueLen - KeyValueDB.RoundToAllocationGranularity(valueLen - MaxValueLenInline)));
}
static ObjectDB CreateDb()
{
var fc = new OnDiskFileCollection(".");
var lowDb = new KeyValueDB(fc, new NoCompressionStrategy());
var db = new ObjectDB();
db.Open(lowDb, true);
return(db);
}
public void CreateEmptyDatabase()
{
using (var stream = CreateTestStream())
{
using (IKeyValueDB db = new KeyValueDB())
{
Assert.IsTrue(db.Open(stream, false));
}
}
}
public void EmptyWritingTransaction()
{
using (var fileCollection = new InMemoryFileCollection())
using (IKeyValueDB db = new KeyValueDB(fileCollection))
{
using (var tr = db.StartWritingTransaction().Result)
{
tr.Commit();
}
}
}
public void CompactorShouldNotBePessimistDespiteRunningTransactions()
{
using var fileCollection = new InMemoryFileCollection();
var options = new KeyValueDBOptions
{
Compression = new NoCompressionStrategy(),
FileCollection = fileCollection,
FileSplitSize = 8096,
CompactorScheduler = CompactorScheduler.Instance,
};
using var kvDb = new KeyValueDB(options);
for (var i = 0; i < 100; i++)
{
using var tr = kvDb.StartWritingTransaction().Result;
var key = new byte[4];
BTDB.Buffer.PackUnpack.PackInt32BE(key, 0, i);
tr.CreateOrUpdateKeyValue(key, new byte[2000]);
tr.Commit();
}
kvDb.Compact(new CancellationToken());
var fileCountAfterFirstCompaction = fileCollection.GetCount();
using (kvDb.StartReadOnlyTransaction())
{
for (var i = 0; i < 50; i++)
{
using var tr = kvDb.StartWritingTransaction().Result;
var key = new byte[4];
BTDB.Buffer.PackUnpack.PackInt32BE(key, 0, i * 2);
tr.FindExactKey(key);
tr.EraseCurrent();
tr.Commit();
}
while (kvDb.Compact(new CancellationToken()))
{
}
Assert.InRange(fileCollection.GetCount(), fileCountAfterFirstCompaction + 2, fileCountAfterFirstCompaction + 50);
}
for (var i = 0; i < 4; i++)
{
using var tr = kvDb.StartWritingTransaction().Result;
var key = new byte[4];
BTDB.Buffer.PackUnpack.PackInt32BE(key, 0, i);
tr.CreateOrUpdateKeyValue(key, new byte[2000]);
tr.Commit();
}
while (kvDb.Compact(new CancellationToken()))
{
}
Assert.InRange(fileCollection.GetCount(), fileCountAfterFirstCompaction / 3, 2 * fileCountAfterFirstCompaction / 3);
}
public void ReadOnlyTransactionThrowsOnWriteAccess()
{
using (var fileCollection = new InMemoryFileCollection())
using (IKeyValueDB db = new KeyValueDB(fileCollection))
{
using (var tr = db.StartReadOnlyTransaction())
{
Assert.Throws <BTDBTransactionRetryException>(() => tr.CreateKey(new byte[1]));
}
}
}
public void EmptyWritingTransaction()
{
using (var fileCollection = new InMemoryFileCollection())
using (IKeyValueDB db = new KeyValueDB(fileCollection))
{
using (var tr = db.StartWritingTransaction().Result)
{
tr.Commit();
}
}
}
public KeyValueDBTransaction(KeyValueDB keyValueDB, IBTreeRootNode btreeRoot, bool writting, bool readOnly)
{
_preapprovedWritting = writting;
_readOnly = readOnly;
_keyValueDB = keyValueDB;
_btreeRoot = btreeRoot;
_prefix = BitArrayManipulation.EmptyByteArray;
_prefixKeyStart = 0;
_prefixKeyCount = -1;
_keyIndex = -1;
_keyValueDB.StartedUsingBTreeRoot(_btreeRoot);
}
public void FirstTransaction()
{
using (var fileCollection = new InMemoryFileCollection())
using (IKeyValueDB db = new KeyValueDB(fileCollection))
{
using (var tr = db.StartTransaction())
{
Assert.True(tr.CreateOrUpdateKeyValue(ByteBuffer.NewAsync(_key1), ByteBuffer.NewAsync(new byte[0])));
tr.Commit();
}
}
}
public void CanGetSizeOfPair()
{
using (var fileCollection = new InMemoryFileCollection())
using (IKeyValueDB db = new KeyValueDB(fileCollection))
{
using (var tr = db.StartTransaction())
{
tr.CreateOrUpdateKeyValue(ByteBuffer.NewAsync(_key1), ByteBuffer.NewAsync(new byte[1]));
var s = tr.GetStorageSizeOfCurrentKey();
Assert.Equal((uint)_key1.Length, s.Key);
Assert.Equal(1u, s.Value);
}
}
}
static void Main(string[] args)
{
if (args.Length < 1)
{
Console.WriteLine("Need to have just one parameter with directory of ObjectDB");
return;
}
using (var dfc = new OnDiskFileCollection(args[0]))
using (var kdb = new KeyValueDB(dfc))
using (var odb = new ObjectDB())
{
odb.Open(kdb, false);
using (var tr = odb.StartTransaction())
{
var visitor = new ToStringVisitor();
var iterator = new ODBIterator(tr, visitor);
iterator.Iterate();
var text = visitor.ToString();
Console.WriteLine(text);
}
}
}
public void AddingContinueToSameFileAfterReopen()
{
using (var fileCollection = new InMemoryFileCollection())
{
using (IKeyValueDB db = new KeyValueDB(fileCollection))
{
using (var tr = db.StartTransaction())
{
tr.CreateOrUpdateKeyValue(_key1, _key1);
tr.Commit();
}
}
using (IKeyValueDB db = new KeyValueDB(fileCollection))
{
using (var tr = db.StartTransaction())
{
tr.CreateOrUpdateKeyValue(Key2, Key2);
tr.Commit();
}
Console.WriteLine(db.CalcStats());
}
Assert.Equal(2u, fileCollection.GetCount()); // Log + Index
}
}
public void FindKeyWithPreferPreviousKeyWorks()
{
const int keyCount = 10000;
using (var fileCollection = new InMemoryFileCollection())
using (IKeyValueDB db = new KeyValueDB(fileCollection))
{
using (var tr = db.StartTransaction())
{
var key = new byte[100];
for (int i = 0; i < keyCount; i++)
{
key[0] = (byte)(i / 256);
key[1] = (byte)(i % 256);
tr.CreateKey(key);
}
tr.Commit();
}
using (var tr = db.StartTransaction())
{
var key = new byte[101];
for (int i = 0; i < keyCount; i++)
{
key[0] = (byte)(i / 256);
key[1] = (byte)(i % 256);
var findKeyResult = tr.Find(ByteBuffer.NewSync(key));
Assert.Equal(FindResult.Previous, findKeyResult);
Assert.Equal(i, tr.GetKeyIndex());
}
}
using (var tr = db.StartTransaction())
{
var key = new byte[99];
for (int i = 0; i < keyCount; i++)
{
key[0] = (byte)(i / 256);
key[1] = (byte)(i % 256);
var findKeyResult = tr.Find(ByteBuffer.NewSync(key));
if (i == 0)
{
Assert.Equal(FindResult.Next, findKeyResult);
Assert.Equal(i, tr.GetKeyIndex());
}
else
{
Assert.Equal(FindResult.Previous, findKeyResult);
Assert.Equal(i - 1, tr.GetKeyIndex());
}
}
}
}
}
public void SimpleFindNextKeyWorks()
{
using (var fileCollection = new InMemoryFileCollection())
using (IKeyValueDB db = new KeyValueDB(fileCollection))
{
using (var tr1 = db.StartTransaction())
{
tr1.CreateKey(_key1);
tr1.CreateKey(Key2);
tr1.CreateKey(_key3);
tr1.Commit();
}
using (var tr2 = db.StartTransaction())
{
Assert.True(tr2.FindExactKey(_key3));
Assert.True(tr2.FindNextKey());
Assert.Equal(Key2, tr2.GetKeyAsByteArray());
Assert.False(tr2.FindNextKey());
}
}
}
public void AdvancedFindPreviousAndNextKeyWorks()
{
using (var fileCollection = new InMemoryFileCollection())
using (IKeyValueDB db = new KeyValueDB(fileCollection))
{
var key = new byte[2];
const int keysCreated = 10000;
using (var tr = db.StartTransaction())
{
for (int i = 0; i < keysCreated; i++)
{
key[0] = (byte)(i / 256);
key[1] = (byte)(i % 256);
tr.CreateKey(key);
}
tr.Commit();
}
using (var tr = db.StartTransaction())
{
Assert.Equal(-1, tr.GetKeyIndex());
tr.FindExactKey(key);
Assert.Equal(keysCreated - 1, tr.GetKeyIndex());
for (int i = 1; i < keysCreated; i++)
{
Assert.True(tr.FindPreviousKey());
Assert.Equal(keysCreated - 1 - i, tr.GetKeyIndex());
}
Assert.False(tr.FindPreviousKey());
Assert.Equal(-1, tr.GetKeyIndex());
for (int i = 0; i < keysCreated; i++)
{
Assert.True(tr.FindNextKey());
Assert.Equal(i, tr.GetKeyIndex());
}
Assert.False(tr.FindNextKey());
Assert.Equal(-1, tr.GetKeyIndex());
}
}
}
public void SetKeyIndexWorks()
{
using (var fileCollection = new InMemoryFileCollection())
using (IKeyValueDB db = new KeyValueDB(fileCollection))
{
var key = new byte[2];
const int keysCreated = 10000;
using (var tr = db.StartTransaction())
{
for (int i = 0; i < keysCreated; i++)
{
key[0] = (byte)(i / 256);
key[1] = (byte)(i % 256);
tr.CreateKey(key);
}
tr.Commit();
}
using (var tr = db.StartTransaction())
{
Assert.False(tr.SetKeyIndex(keysCreated));
for (int i = 0; i < keysCreated; i += 5)
{
Assert.True(tr.SetKeyIndex(i));
key = tr.GetKeyAsByteArray();
Assert.Equal((byte)(i / 256), key[0]);
Assert.Equal((byte)(i % 256), key[1]);
Assert.Equal(i, tr.GetKeyIndex());
}
}
}
}
public void MoreComplexReopen()
{
using (var fileCollection = new InMemoryFileCollection())
{
using (IKeyValueDB db = new KeyValueDB(fileCollection))
{
for (int i = 0; i < 100; i++)
{
var key = new byte[100];
using (var tr = db.StartTransaction())
{
key[0] = (byte)(i / 256);
key[1] = (byte)(i % 256);
Assert.True(tr.CreateOrUpdateKeyValue(key, key));
tr.Commit();
}
}
using (var tr = db.StartTransaction())
{
tr.SetKeyIndex(0);
tr.EraseCurrent();
tr.EraseRange(1, 3);
tr.Commit();
}
}
using (IKeyValueDB db = new KeyValueDB(fileCollection))
{
using (var tr = db.StartTransaction())
{
var key = new byte[100];
key[1] = 1;
Assert.True(tr.FindExactKey(key));
tr.FindNextKey();
Assert.Equal(5, tr.GetKeyAsByteArray()[1]);
Assert.Equal(96, tr.GetKeyValueCount());
}
}
}
}
public void BiggerKey(int prefixLength, int offsetKey, int keyLength)
{
var prefix = new byte[prefixLength];
var keyb = new byte[offsetKey + keyLength];
for (int i = offsetKey; i < offsetKey + keyLength; i++) keyb[i] = (byte)i;
var key = ByteBuffer.NewAsync(keyb, offsetKey, keyLength);
using (var fileCollection = new InMemoryFileCollection())
using (IKeyValueDB db = new KeyValueDB(fileCollection))
{
using (var tr1 = db.StartTransaction())
{
tr1.SetKeyPrefix(prefix);
tr1.CreateOrUpdateKeyValue(key, ByteBuffer.NewEmpty());
tr1.Commit();
}
using (var tr2 = db.StartTransaction())
{
tr2.SetKeyPrefix(prefix);
Assert.True(tr2.FindExactKey(key.ToByteArray()));
Assert.Equal(key.ToByteArray(), tr2.GetKeyAsByteArray());
}
}
}
public void PrefixWithFindPrevKeyWorks()
{
using (var fileCollection = new InMemoryFileCollection())
using (IKeyValueDB db = new KeyValueDB(fileCollection))
{
using (var tr = db.StartTransaction())
{
tr.CreateKey(_key1);
tr.CreateKey(Key2);
tr.SetKeyPrefix(ByteBuffer.NewAsync(Key2, 0, 1));
Assert.True(tr.FindFirstKey());
Assert.False(tr.FindPreviousKey());
tr.Commit();
}
}
}
public void SimpleEraseCurrentWorks()
{
using (var fileCollection = new InMemoryFileCollection())
using (IKeyValueDB db = new KeyValueDB(fileCollection))
{
using (var tr = db.StartTransaction())
{
tr.CreateKey(_key1);
tr.CreateKey(Key2);
tr.CreateKey(_key3);
tr.EraseCurrent();
Assert.True(tr.FindFirstKey());
Assert.Equal(_key1, tr.GetKeyAsByteArray());
Assert.True(tr.FindNextKey());
Assert.Equal(Key2, tr.GetKeyAsByteArray());
Assert.False(tr.FindNextKey());
Assert.Equal(2, tr.GetKeyValueCount());
}
}
}
void AdvancedEraseRangeWorks(int createKeys, int removeStart, int removeCount)
{
using (var fileCollection = new InMemoryFileCollection())
using (IKeyValueDB db = new KeyValueDB(fileCollection))
{
var key = new byte[2];
using (var tr = db.StartTransaction())
{
for (int i = 0; i < createKeys; i++)
{
key[0] = (byte)(i / 256);
key[1] = (byte)(i % 256);
tr.CreateKey(key);
}
tr.Commit();
}
using (var tr = db.StartTransaction())
{
tr.EraseRange(removeStart, removeStart + removeCount - 1);
Assert.Equal(createKeys - removeCount, tr.GetKeyValueCount());
tr.Commit();
}
using (var tr = db.StartTransaction())
{
Assert.Equal(createKeys - removeCount, tr.GetKeyValueCount());
for (int i = 0; i < createKeys; i++)
{
key[0] = (byte)(i / 256);
key[1] = (byte)(i % 256);
if (i >= removeStart && i < removeStart + removeCount)
{
Assert.False(tr.FindExactKey(key), $"{i} should be removed");
}
else
{
Assert.True(tr.FindExactKey(key), $"{i} should be found");
}
}
}
}
}
public void ALotOf5KbTransactionsWorks()
{
using (var fileCollection = new InMemoryFileCollection())
using (IKeyValueDB db = new KeyValueDB(fileCollection))
{
for (int i = 0; i < 5000; i++)
{
var key = new byte[5000];
using (var tr = db.StartTransaction())
{
key[0] = (byte)(i / 256);
key[1] = (byte)(i % 256);
Assert.True(tr.CreateKey(key));
tr.Commit();
}
}
}
}
public void CompressibleValueLoad()
{
using (var fileCollection = new InMemoryFileCollection())
{
using (IKeyValueDB db = new KeyValueDB(fileCollection))
{
using (var tr = db.StartTransaction())
{
tr.CreateOrUpdateKeyValue(_key1, new byte[1000]);
Assert.Equal(new byte[1000], tr.GetValueAsByteArray());
tr.Commit();
}
}
}
}
public void StartWritingTransactionWorks()
{
using (var fileCollection = new InMemoryFileCollection())
using (IKeyValueDB db = new KeyValueDB(fileCollection))
{
var tr1 = db.StartWritingTransaction().Result;
var tr2Task = db.StartWritingTransaction();
var task = Task.Factory.StartNew(() =>
{
var tr2 = tr2Task.Result;
Assert.True(tr2.FindExactKey(_key1));
tr2.CreateKey(Key2);
tr2.Commit();
tr2.Dispose();
});
tr1.CreateKey(_key1);
tr1.Commit();
tr1.Dispose();
task.Wait(1000);
using (var tr = db.StartTransaction())
{
Assert.True(tr.FindExactKey(_key1));
Assert.True(tr.FindExactKey(Key2));
}
}
}
public void RepairsOnReopen()
{
using (var fileCollection = new InMemoryFileCollection())
{
using (IKeyValueDB db = new KeyValueDB(fileCollection))
{
using (var tr = db.StartTransaction())
{
tr.CreateKey(_key1);
tr.Commit();
}
using (var tr = db.StartTransaction())
{
tr.CreateKey(Key2);
tr.Commit();
}
using (var tr = db.StartTransaction())
{
tr.CreateKey(_key3);
// rollback
}
using (IKeyValueDB db2 = new KeyValueDB(fileCollection))
{
using (var tr = db2.StartTransaction())
{
Assert.True(tr.FindExactKey(_key1));
Assert.True(tr.FindExactKey(Key2));
Assert.False(tr.FindExactKey(_key3));
}
}
}
using (IKeyValueDB db = new KeyValueDB(fileCollection))
{
using (var tr = db.StartTransaction())
{
Assert.True(tr.FindExactKey(_key1));
Assert.True(tr.FindExactKey(Key2));
Assert.False(tr.FindExactKey(_key3));
}
}
}
}
public void MultipleTransactions2(int transactionCount)
{
using (var fileCollection = new InMemoryFileCollection())
using (IKeyValueDB db = new KeyValueDB(fileCollection))
{
var key = new byte[2 + transactionCount * 10];
for (int i = 0; i < transactionCount; i++)
{
key[0] = (byte)((transactionCount - i) / 256);
key[1] = (byte)((transactionCount - i) % 256);
using (var tr1 = db.StartTransaction())
{
tr1.CreateOrUpdateKeyValue(ByteBuffer.NewSync(key, 0, 2 + i * 10), ByteBuffer.NewEmpty());
if (i % 100 == 0 || i == transactionCount - 1)
{
for (int j = 0; j < i; j++)
{
key[0] = (byte)((transactionCount - j) / 256);
key[1] = (byte)((transactionCount - j) % 256);
Assert.Equal(FindResult.Exact, tr1.Find(ByteBuffer.NewSync(key, 0, 2 + j * 10)));
}
}
tr1.Commit();
}
}
}
}
public void TwoTransactions()
{
using (var fileCollection = new InMemoryFileCollection())
using (IKeyValueDB db = new KeyValueDB(fileCollection))
{
using (var tr1 = db.StartTransaction())
{
tr1.CreateKey(_key1);
tr1.Commit();
}
using (var tr2 = db.StartTransaction())
{
tr2.CreateKey(Key2);
Assert.True(tr2.FindExactKey(_key1));
Assert.True(tr2.FindExactKey(Key2));
Assert.False(tr2.FindExactKey(_key3));
tr2.Commit();
}
using (var tr3 = db.StartTransaction())
{
Assert.True(tr3.FindExactKey(_key1));
Assert.True(tr3.FindExactKey(Key2));
Assert.False(tr3.FindExactKey(_key3));
}
}
}
public void SimplePrefixWorks()
{
using (var fileCollection = new InMemoryFileCollection())
using (IKeyValueDB db = new KeyValueDB(fileCollection))
{
using (var tr = db.StartTransaction())
{
tr.CreateKey(_key1);
tr.CreateKey(Key2);
tr.CreateKey(_key3);
Assert.Equal(3, tr.GetKeyValueCount());
tr.SetKeyPrefix(ByteBuffer.NewAsync(_key1, 0, 3));
Assert.Equal(2, tr.GetKeyValueCount());
tr.FindFirstKey();
Assert.Equal(new byte[0], tr.GetKeyAsByteArray());
tr.FindLastKey();
Assert.Equal(_key3.Skip(3).ToArray(), tr.GetKeyAsByteArray());
tr.Commit();
}
}
}
public void TwoEmptyWriteTransactionsWithNestedWaiting()
{
using (var fileCollection = new InMemoryFileCollection())
using (IKeyValueDB db = new KeyValueDB(fileCollection))
{
Task<IKeyValueDBTransaction> trOuter;
using (var tr = db.StartWritingTransaction().Result)
{
trOuter = db.StartWritingTransaction();
tr.Commit();
}
using (var tr = trOuter.Result)
{
tr.Commit();
}
}
}
public void FindLastKeyWorks()
{
using (var fileCollection = new InMemoryFileCollection())
using (IKeyValueDB db = new KeyValueDB(fileCollection))
{
using (var tr = db.StartTransaction())
{
Assert.False(tr.FindLastKey());
tr.CreateKey(_key1);
tr.CreateKey(Key2);
tr.CreateKey(_key3);
Assert.True(tr.FindLastKey());
Assert.Equal(Key2, tr.GetKeyAsByteArray());
tr.Commit();
}
}
}
public void CreateOrUpdateKeyValueWorks(int length)
{
var valbuf = new byte[length];
new Random(0).NextBytes(valbuf);
using (var fileCollection = new InMemoryFileCollection())
using (IKeyValueDB db = new KeyValueDB(fileCollection))
{
using (var tr1 = db.StartTransaction())
{
Assert.True(tr1.CreateOrUpdateKeyValueUnsafe(_key1, valbuf));
Assert.False(tr1.CreateOrUpdateKeyValueUnsafe(_key1, valbuf));
Assert.True(tr1.CreateOrUpdateKeyValueUnsafe(Key2, valbuf));
tr1.Commit();
}
using (var tr2 = db.StartTransaction())
{
Assert.True(tr2.FindExactKey(_key1));
var valbuf2 = tr2.GetValueAsByteArray();
for (int i = 0; i < length; i++)
{
if (valbuf[i] != valbuf2[i])
Assert.Equal(valbuf[i], valbuf2[i]);
}
Assert.True(tr2.FindExactKey(Key2));
valbuf2 = tr2.GetValueAsByteArray();
for (int i = 0; i < length; i++)
{
if (valbuf[i] != valbuf2[i])
Assert.Equal(valbuf[i], valbuf2[i]);
}
}
}
}
public void ReadOnlyTransactionThrowsOnWriteAccess()
{
using (var fileCollection = new InMemoryFileCollection())
using (IKeyValueDB db = new KeyValueDB(fileCollection))
{
using (var tr = db.StartReadOnlyTransaction())
{
Assert.Throws<BTDBTransactionRetryException>(() => tr.CreateKey(new byte[1]));
}
}
}
internal Compactor(KeyValueDB keyValueDB, CancellationToken cancellation)
{
_keyValueDB = keyValueDB;
_cancellation = cancellation;
}
public void SetKeyPrefixInOneTransaction()
{
using (var fileCollection = new InMemoryFileCollection())
using (IKeyValueDB db = new KeyValueDB(fileCollection))
{
var key = new byte[5];
var value = new byte[100];
var rnd = new Random();
using (var tr = db.StartTransaction())
{
for (byte i = 0; i < 100; i++)
{
key[0] = i;
for (byte j = 0; j < 100; j++)
{
key[4] = j;
rnd.NextBytes(value);
tr.CreateOrUpdateKeyValue(key, value);
}
}
tr.Commit();
}
using (var tr = db.StartTransaction())
{
for (byte i = 0; i < 100; i++)
{
key[0] = i;
tr.SetKeyPrefix(ByteBuffer.NewSync(key, 0, 4));
Assert.Equal(100, tr.GetKeyValueCount());
}
}
}
}
