uint ITableInfoResolver.GetLastPesistedVersion(uint id)
{
using (var tr = _keyValueDB.StartTransaction())
{
tr.SetKeyPrefix(TableVersionsPrefix);
var key = new byte[PackUnpack.LengthVUInt(id) + 1];
var ofs = 0;
PackUnpack.PackVUInt(key, ref ofs, id);
key[ofs] = 0xff;
if (tr.FindKey(key, 0, key.Length, FindKeyStrategy.PreferPrevious) == FindKeyResult.NotFound)
{
return(0);
}
var key2 = tr.ReadKey();
if (key2.Length < ofs)
{
return(0);
}
if (BitArrayManipulation.CompareByteArray(key, 0, ofs, key2, 0, ofs) != 0)
{
return(0);
}
return(checked ((uint)PackUnpack.UnpackVUInt(key2, ref ofs)));
}
}
void CheckKeySequence(int keys)
{
var sw = Stopwatch.StartNew();
var key = new byte[8];
using (var fileCollection = CreateTestFileCollection())
{
using (IKeyValueDB db = CreateKeyValueDB(fileCollection, new NoCompressionStrategy()))
{
using (var tr = db.StartTransaction())
{
for (int i = 0; i < keys; i++)
{
int o = 0;
PackUnpack.PackVUInt(key, ref o, (uint)i);
tr.Find(ByteBuffer.NewSync(key, 0, o));
}
tr.Commit();
}
}
}
Console.WriteLine("CheckSequence:" + sw.Elapsed.TotalMilliseconds);
}
void CreateTestDB(int keys)
{
var rnd = new Random(1234);
using (var fileCollection = CreateTestFileCollection())
{
using (IKeyValueDB db = CreateKeyValueDB(fileCollection))
{
using (var tr = db.StartTransaction())
{
for (int i = 0; i < keys; i++)
{
var key = new byte[rnd.Next(10, 50)];
rnd.NextBytes(key);
var value = new byte[rnd.Next(50, 500)];
rnd.NextBytes(value);
tr.CreateOrUpdateKeyValueUnsafe(key, value);
}
tr.Commit();
}
}
using (IKeyValueDB db = CreateKeyValueDB(fileCollection))
{
}
}
}
void CreateRandomKeySequence(int keys)
{
var sw = Stopwatch.StartNew();
var rnd = new Random(1234);
using (var fileCollection = CreateTestFileCollection())
{
using (IKeyValueDB db = CreateKeyValueDB(fileCollection, new NoCompressionStrategy()))
{
db.DurableTransactions = true;
using (var tr = db.StartTransaction())
{
for (int i = 0; i < keys; i++)
{
var key = new byte[rnd.Next(10, 50)];
rnd.NextBytes(key);
var value = new byte[rnd.Next(50, 500)];
rnd.NextBytes(value);
tr.CreateOrUpdateKeyValueUnsafe(key, value);
}
tr.Commit();
}
}
}
Console.WriteLine("CreateSequence:" + sw.Elapsed.TotalMilliseconds);
}
public void Open(IKeyValueDB keyValueDB, bool dispose)
{
if (keyValueDB == null)
{
throw new ArgumentNullException(nameof(keyValueDB));
}
_keyValueDB = keyValueDB;
_dispose = dispose;
_tableInfoResolver = new TableInfoResolver(keyValueDB, this);
_tablesInfo = new TablesInfo(_tableInfoResolver);
_relationsInfoResolver = new RelationInfoResolver(this);
_relationsInfo = new RelationsInfo(_relationsInfoResolver);
_lastObjId = 0;
using (var tr = _keyValueDB.StartTransaction())
{
tr.SetKeyPrefix(AllObjectsPrefix);
if (tr.FindLastKey())
{
_lastObjId = (long)new KeyValueDBKeyReader(tr).ReadVUInt64();
}
_tablesInfo.LoadTables(LoadTablesEnum(tr));
_relationsInfo.LoadRelations(LoadRelationNamesEnum(tr));
tr.SetKeyPrefix(null);
if (tr.FindExactKey(LastDictIdKey))
{
_lastDictId = new ByteArrayReader(tr.GetValueAsByteArray()).ReadVUInt64();
}
}
}
public void Open(IKeyValueDB keyValueDB, bool dispose, DBOptions options)
{
_keyValueDB = keyValueDB ?? throw new ArgumentNullException(nameof(keyValueDB));
_dispose = dispose;
_type2Name = options.CustomType2NameRegistry ?? new Type2NameRegistry();
_polymorphicTypesRegistry = new PolymorphicTypesRegistry();
AutoRegisterTypes = options.AutoRegisterType;
ActualOptions = options;
SymmetricCipher = options.SymmetricCipher ?? new InvalidSymmetricCipher();
_tableInfoResolver = new TableInfoResolver(keyValueDB, this);
_tablesInfo = new TablesInfo(_tableInfoResolver);
_relationsInfoResolver = new RelationInfoResolver(this);
_relationsInfo = new RelationsInfo(_relationsInfoResolver);
using var tr = _keyValueDB.StartTransaction();
_lastObjId = (long)tr.GetUlong(0);
_lastDictId = tr.GetUlong(1);
if (_lastObjId == 0)
{
if (tr.FindLastKey(AllObjectsPrefix))
{
_lastObjId = (long)PackUnpack.UnpackVUInt(tr.GetKey().Slice(AllObjectsPrefixLen));
}
}
_tablesInfo.LoadTables(LoadTablesEnum(tr));
_relationsInfo.LoadRelations(LoadRelationNamesEnum(tr));
if (_lastDictId == 0)
{
if (tr.FindExactKey(LastDictIdKey))
{
_lastDictId = PackUnpack.UnpackVUInt(tr.GetValue());
}
}
}
void ReadAllValues(IKeyValueDB kv)
{
using var tr = kv.StartTransaction();
tr.InvalidateCurrentKey();
while (tr.FindNextKey(ReadOnlySpan <byte> .Empty))
{
tr.GetValue();
}
}
void DoWork5(bool alsoDoReads)
{
_sw.Restart();
long pureDataLength = 0;
using (var fileCollection = CreateTestFileCollection())
using (IKeyValueDB db = CreateKeyValueDB(fileCollection))
{
for (int i = 0; i < 200; i++)
{
long pureDataLengthPrevTr = pureDataLength;
using (var tr = db.StartTransaction())
{
for (int j = 0; j < 200; j++)
{
tr.CreateOrUpdateKeyValue(ByteBuffer.NewAsync(new[] { (byte)j, (byte)i }), ByteBuffer.NewAsync(new byte[1 + i * j]));
pureDataLength += 2 + 1 + i * j;
}
if (alsoDoReads)
{
using (var trCheck = db.StartTransaction())
{
long pureDataLengthCheck = 0;
while (trCheck.FindNextKey())
{
pureDataLengthCheck += trCheck.GetKey().Length + trCheck.GetValue().Length;
}
if (pureDataLengthCheck != pureDataLengthPrevTr)
{
throw new Exception("Transactions are not in serializable mode");
}
}
}
tr.Commit();
}
}
}
_sw.Stop();
Console.WriteLine("Pure data length: {0}", pureDataLength);
Console.WriteLine("Time: {0,15}ms", _sw.Elapsed.TotalMilliseconds);
}
public IKeyValueDBTransaction StartTransaction()
{
var result = new KeyValueDBReplayTransactionProxy(_db.StartTransaction(), _log, ref _trCounter);
lock (_log)
{
_log.WriteUInt8((byte)KVReplayOperation.StartTransaction);
_log.WriteVUInt32(result.TrIndex);
_log.FlushBuffer();
}
return(result);
}
void ReadAllValues(IKeyValueDB kv)
{
using (var tr = kv.StartTransaction())
{
if (tr.FindFirstKey())
{
do
{
tr.GetValueAsByteArray();
} while (tr.FindNextKey());
}
}
}
uint ITableInfoResolver.GetLastPersistedVersion(uint id)
{
using (var tr = _keyValueDB.StartTransaction())
{
tr.SetKeyPrefix(TableVersionsPrefix);
var key = TableInfo.BuildKeyForTableVersions(id, uint.MaxValue);
if (tr.Find(ByteBuffer.NewSync(key)) == FindResult.NotFound)
{
return(0);
}
var key2 = tr.GetKeyAsByteArray();
var ofs = PackUnpack.LengthVUInt(id);
if (key2.Length < ofs)
{
return(0);
}
if (BitArrayManipulation.CompareByteArray(key, ofs, key2, ofs) != 0)
{
return(0);
}
return(checked ((uint)PackUnpack.UnpackVUInt(key2, ref ofs)));
}
}
public void Open(IKeyValueDB keyValueDB, bool dispose)
{
if (keyValueDB == null) throw new ArgumentNullException("keyValueDB");
_keyValueDB = keyValueDB;
_dispose = dispose;
_tableInfoResolver = new TableInfoResolver(keyValueDB, this);
_tablesInfo = new TablesInfo(_tableInfoResolver);
_lastObjId = 0;
using (var tr = _keyValueDB.StartTransaction())
{
tr.SetKeyPrefix(AllObjectsPrefix);
if (tr.FindLastKey())
{
_lastObjId = (long)new KeyValueDBKeyReader(tr).ReadVUInt64();
}
_tablesInfo.LoadTables(LoadTablesEnum(tr));
}
}
public void Open(IKeyValueDB keyValueDB, bool dispose, DBOptions options)
{
if (keyValueDB == null)
{
throw new ArgumentNullException(nameof(keyValueDB));
}
_keyValueDB = keyValueDB;
_dispose = dispose;
_type2Name = options.CustomType2NameRegistry ?? new Type2NameRegistry();
_polymorphicTypesRegistry = new PolymorphicTypesRegistry();
AutoRegisterTypes = options.AutoRegisterType;
ActualOptions = options;
_symmetricCipher = options.SymmetricCipher ?? new InvalidSymmetricCipher();
_tableInfoResolver = new TableInfoResolver(keyValueDB, this);
_tablesInfo = new TablesInfo(_tableInfoResolver);
_relationsInfoResolver = new RelationInfoResolver(this);
_relationsInfo = new RelationsInfo(_relationsInfoResolver);
using (var tr = _keyValueDB.StartTransaction())
{
_lastObjId = (long)tr.GetUlong(0);
_lastDictId = tr.GetUlong(1);
if (_lastObjId == 0)
{
tr.SetKeyPrefix(AllObjectsPrefix);
if (tr.FindLastKey())
{
_lastObjId = (long)new KeyValueDBKeyReader(tr).ReadVUInt64();
}
}
_tablesInfo.LoadTables(LoadTablesEnum(tr));
_relationsInfo.LoadRelations(LoadRelationNamesEnum(tr));
if (_lastDictId == 0)
{
tr.SetKeyPrefix(null);
if (tr.FindExactKey(LastDictIdKey))
{
_lastDictId = new ByteArrayReader(tr.GetValueAsByteArray()).ReadVUInt64();
}
}
}
}
void CheckDBTest(int keys)
{
var rnd = new Random(1234);
using (var fileCollection = OpenTestFileCollection())
{
using (IKeyValueDB db = CreateKeyValueDB(fileCollection))
{
using (var tr = db.StartTransaction())
{
if (tr.GetKeyValueCount() != keys)
{
throw new Exception("KeyCount does not match");
}
for (var i = 0; i < keys; i++)
{
var key = new byte[rnd.Next(10, 50)];
rnd.NextBytes(key);
var value = new byte[rnd.Next(50, 500)];
rnd.NextBytes(value);
if (!tr.FindExactKey(key))
{
throw new Exception("Key not found");
}
var value2 = tr.GetValueAsByteArray();
if (value.Length != value2.Length)
{
throw new Exception("value length different");
}
for (var j = 0; j < value.Length; j++)
{
if (value[j] != value2[j])
{
throw new Exception("value different");
}
}
}
}
}
}
}
public void Open(IKeyValueDB keyValueDB, bool dispose)
{
if (keyValueDB == null)
{
throw new ArgumentNullException("keyValueDB");
}
_keyValueDB = keyValueDB;
_dispose = dispose;
_tableInfoResolver = new TableInfoResolver(keyValueDB, this);
_tablesInfo = new TablesInfo(_tableInfoResolver);
_lastObjId = 0;
using (var tr = _keyValueDB.StartTransaction())
{
tr.SetKeyPrefix(AllObjectsPrefix);
if (tr.FindLastKey())
{
_lastObjId = (long)new KeyValueDBKeyReader(tr).ReadVUInt64();
}
_tablesInfo.LoadTables(LoadTablesEnum(tr));
}
}
public void Replay()
{
while (!_reader.Eof)
{
var operation = (KVReplayOperation)_reader.ReadUInt8();
Console.WriteLine(operation);
uint tri;
IKeyValueDBTransaction tr;
int valOfs;
int valLen;
byte[] valBuf;
int keyLen;
int keyOfs;
byte[] keyBuf;
switch (operation)
{
case KVReplayOperation.Open:
var initialStreamSize = _reader.ReadVUInt64();
_dbstream = new MemoryPositionLessStream();
var buf = new byte[4096];
ulong pos = 0;
while (pos < initialStreamSize)
{
var r = (int)Math.Min((ulong)buf.Length, initialStreamSize - pos);
_reader.ReadBlock(buf, 0, r);
_dbstream.Write(buf, 0, r, pos);
pos += (ulong)r;
}
_db.Open(_dbstream, true);
break;
case KVReplayOperation.KeyValueDBDispose:
_db.Dispose();
break;
case KVReplayOperation.StartTransaction:
tr = _db.StartTransaction();
tri = _reader.ReadVUInt32();
_trs[tri] = tr;
break;
case KVReplayOperation.StartWritingTransaction:
tr = _db.StartWritingTransaction().Result;
tri = _reader.ReadVUInt32();
_trs[tri] = tr;
break;
case KVReplayOperation.CalculateStats:
tri = _reader.ReadVUInt32();
tr = _trs[tri];
tr.CalculateStats();
break;
case KVReplayOperation.SetKeyPrefix:
tri = _reader.ReadVUInt32();
tr = _trs[tri];
int prefixLen = _reader.ReadVInt32();
int prefixOfs = _reader.ReadVInt32();
var prefixBuf = new byte[prefixOfs + prefixLen];
_reader.ReadBlock(prefixBuf, prefixOfs, prefixLen);
tr.SetKeyPrefix(prefixBuf, prefixOfs, prefixLen);
break;
case KVReplayOperation.FindKey:
tri = _reader.ReadVUInt32();
tr = _trs[tri];
keyLen = _reader.ReadVInt32();
keyOfs = _reader.ReadVInt32();
keyBuf = new byte[keyOfs + keyLen];
_reader.ReadBlock(keyBuf, keyOfs, keyLen);
var strategy = (FindKeyStrategy)_reader.ReadVUInt32();
tr.FindKey(keyBuf, keyOfs, keyLen, strategy);
break;
case KVReplayOperation.GetKeyIndex:
tri = _reader.ReadVUInt32();
tr = _trs[tri];
tr.GetKeyIndex();
break;
case KVReplayOperation.SetValue:
tri = _reader.ReadVUInt32();
tr = _trs[tri];
valOfs = _reader.ReadVInt32();
valLen = _reader.ReadVInt32();
valBuf = new byte[valOfs + valLen];
_reader.ReadBlock(valBuf, valOfs, valLen);
tr.SetValue(valBuf, valOfs, valLen);
break;
case KVReplayOperation.SetValueSize:
tri = _reader.ReadVUInt32();
tr = _trs[tri];
var valueSize = _reader.ReadVInt64();
tr.SetValueSize(valueSize);
break;
case KVReplayOperation.WriteValue:
tri = _reader.ReadVUInt32();
tr = _trs[tri];
var ofs = _reader.ReadVInt64();
valOfs = _reader.ReadVInt32();
valLen = _reader.ReadVInt32();
valBuf = new byte[valOfs + valLen];
_reader.ReadBlock(valBuf, valOfs, valLen);
tr.WriteValue(ofs, valLen, valBuf, valOfs);
break;
case KVReplayOperation.CreateOrUpdateKeyValue:
tri = _reader.ReadVUInt32();
tr = _trs[tri];
keyLen = _reader.ReadVInt32();
keyOfs = _reader.ReadVInt32();
keyBuf = new byte[keyOfs + keyLen];
_reader.ReadBlock(keyBuf, keyOfs, keyLen);
valLen = _reader.ReadVInt32();
valOfs = _reader.ReadVInt32();
valBuf = new byte[valOfs + valLen];
_reader.ReadBlock(valBuf, valOfs, valLen);
tr.CreateOrUpdateKeyValue(keyBuf, keyOfs, keyLen, valBuf, valOfs, valLen);
break;
case KVReplayOperation.Commit:
tri = _reader.ReadVUInt32();
tr = _trs[tri];
tr.Commit();
break;
case KVReplayOperation.TransactionDispose:
tri = _reader.ReadVUInt32();
tr = _trs[tri];
tr.Dispose();
_trs.Remove(tri);
break;
case KVReplayOperation.EraseCurrent:
tri = _reader.ReadVUInt32();
tr = _trs[tri];
tr.EraseCurrent();
break;
case KVReplayOperation.EraseAll:
tri = _reader.ReadVUInt32();
tr = _trs[tri];
tr.EraseAll();
break;
case KVReplayOperation.FindFirstKey:
tri = _reader.ReadVUInt32();
tr = _trs[tri];
tr.FindFirstKey();
break;
case KVReplayOperation.FindPreviousKey:
tri = _reader.ReadVUInt32();
tr = _trs[tri];
tr.FindPreviousKey();
break;
case KVReplayOperation.FindNextKey:
tri = _reader.ReadVUInt32();
tr = _trs[tri];
tr.FindNextKey();
break;
default:
Console.WriteLine(string.Format("Unimplemented operation {0}({1})", operation, (byte)operation));
throw new NotSupportedException(string.Format("Unimplemented operation {0}({1})", operation, (byte)operation));
}
}
Console.WriteLine("Finish");
}
public IObjectDBTransaction StartTransaction()
{
return(new ObjectDBTransaction(this, _keyValueDB.StartTransaction(), false));
}
static void StartLeakingTransaction(IKeyValueDB db)
{
db.StartTransaction().DescriptionForLeaks = "Leak";
}
public IKeyValueDBTransaction StartTransaction()
{
return(new KeyValueDBTransactionWithCount(_keyValueDB.StartTransaction()));
}
public void HugeTest()
{
const int keyCount = 100;
using (var fileCollection = CreateTestFileCollection())
{
_sw.Start();
using (IKeyValueDB db = CreateKeyValueDB(fileCollection, new NoCompressionStrategy()))
{
var key = new byte[100];
var value = new byte[100000000];
for (int i = 0; i < keyCount; i++)
{
using (var tr = db.StartTransaction())
{
key[0] = (byte)(i / 100);
key[1] = (byte)(i % 100);
value[100] = (byte)(i / 100);
value[200] = (byte)(i % 100);
tr.CreateOrUpdateKeyValue(key, value);
tr.Commit();
}
}
}
_sw.Stop();
Console.WriteLine("Time to create 10GB DB: {0,15}ms", _sw.Elapsed.TotalMilliseconds);
_sw.Restart();
using (IKeyValueDB db = new KeyValueDB(fileCollection))
{
_sw.Stop();
Console.WriteLine("Time to open 10GB DB: {0,15}ms", _sw.Elapsed.TotalMilliseconds);
_sw.Restart();
var key = new byte[100];
for (int i = 0; i < keyCount; i++)
{
using (var tr = db.StartTransaction())
{
key[0] = (byte)(i / 100);
key[1] = (byte)(i % 100);
tr.FindExactKey(key);
var value = tr.GetValueAsByteArray();
if (value[100] != (byte)(i / 100))
{
throw new InvalidDataException();
}
if (value[200] != (byte)(i % 100))
{
throw new InvalidDataException();
}
}
}
_sw.Stop();
Console.WriteLine("Time to read all values 10GB DB: {0,15}ms", _sw.Elapsed.TotalMilliseconds);
Console.WriteLine(db.CalcStats());
}
_sw.Restart();
using (IKeyValueDB db = CreateKeyValueDB(fileCollection))
{
_sw.Stop();
Console.WriteLine("Time to open2 10GB DB: {0,15}ms", _sw.Elapsed.TotalMilliseconds);
_sw.Restart();
var key = new byte[100];
for (int i = 0; i < keyCount; i++)
{
using (var tr = db.StartTransaction())
{
key[0] = (byte)(i / 100);
key[1] = (byte)(i % 100);
tr.FindExactKey(key);
var value = tr.GetValueAsByteArray();
if (value[100] != (byte)(i / 100))
{
throw new InvalidDataException();
}
if (value[200] != (byte)(i % 100))
{
throw new InvalidDataException();
}
}
}
_sw.Stop();
Console.WriteLine("Time to read2 all values 10GB DB: {0,15}ms", _sw.Elapsed.TotalMilliseconds);
Console.WriteLine(db.CalcStats());
}
}
}
public void Open(IKeyValueDB keyValueDB, bool dispose)
{
if (keyValueDB == null) throw new ArgumentNullException(nameof(keyValueDB));
_keyValueDB = keyValueDB;
_dispose = dispose;
_tableInfoResolver = new TableInfoResolver(keyValueDB, this);
_tablesInfo = new TablesInfo(_tableInfoResolver);
_relationsInfoResolver = new RelationInfoResolver(this);
_relationsInfo = new RelationsInfo(_relationsInfoResolver);
_lastObjId = 0;
using (var tr = _keyValueDB.StartTransaction())
{
tr.SetKeyPrefix(AllObjectsPrefix);
if (tr.FindLastKey())
{
_lastObjId = (long)new KeyValueDBKeyReader(tr).ReadVUInt64();
}
_tablesInfo.LoadTables(LoadTablesEnum(tr));
_relationsInfo.LoadRelations(LoadRelationNamesEnum(tr));
tr.SetKeyPrefix(null);
if (tr.FindExactKey(LastDictIdKey))
{
_lastDictId = new ByteArrayReader(tr.GetValueAsByteArray()).ReadVUInt64();
}
}
}