public void Open(IKeyValueDB keyValueDB, bool dispose, DBOptions options)
{
_keyValueDB = keyValueDB ?? throw new ArgumentNullException(nameof(keyValueDB));
_dispose = dispose;
_type2Name = options.CustomType2NameRegistry ?? new Type2NameRegistry();
_polymorphicTypesRegistry = new PolymorphicTypesRegistry();
AutoRegisterTypes = options.AutoRegisterType;
ActualOptions = options;
SymmetricCipher = options.SymmetricCipher ?? new InvalidSymmetricCipher();
_tableInfoResolver = new TableInfoResolver(keyValueDB, this);
_tablesInfo = new TablesInfo(_tableInfoResolver);
_relationsInfoResolver = new RelationInfoResolver(this);
_relationsInfo = new RelationsInfo(_relationsInfoResolver);
using var tr = _keyValueDB.StartTransaction();
_lastObjId = (long)tr.GetUlong(0);
_lastDictId = tr.GetUlong(1);
if (_lastObjId == 0)
{
if (tr.FindLastKey(AllObjectsPrefix))
{
_lastObjId = (long)PackUnpack.UnpackVUInt(tr.GetKey().Slice(AllObjectsPrefixLen));
}
}
_tablesInfo.LoadTables(LoadTablesEnum(tr));
_relationsInfo.LoadRelations(LoadRelationNamesEnum(tr));
if (_lastDictId == 0)
{
if (tr.FindExactKey(LastDictIdKey))
{
_lastDictId = PackUnpack.UnpackVUInt(tr.GetValue());
}
}
}
void OpenDB()
{
_lowDB = new KeyValueDB();
_lowDB.Open(_dbstream, false);
_db = new ObjectDB();
_db.Open(_lowDB, true);
}
public void Dispose()
{
_keyValueDb?.Dispose();
_keyValueDb = null;
_memoryFileCollection?.Dispose();
_memoryFileCollection = null;
}
static Task StartReader(IKeyValueDB lowDb, string id, int sleep, CancellationToken cancellationToken)
{
return(Task.Run(async() =>
{
while (true)
{
var transaction = lowDb.StartReadOnlyTransaction();
transaction.FindFirstKey();
var keyValueCount = transaction.GetKeyValueCount();
for (long kv = 0; kv < keyValueCount; kv++)
{
transaction.GetKey();
transaction.GetValue();
transaction.FindNextKey();
await Task.Delay(sleep, cancellationToken);
if (cancellationToken.IsCancellationRequested)
{
return;
}
}
Console.Write($"{id} ");
}
}, cancellationToken));
}
void CreateTestDB(int keys)
{
var rnd = new Random(1234);
using (var fileCollection = CreateTestFileCollection())
{
using (IKeyValueDB db = CreateKeyValueDB(fileCollection))
{
using (var tr = db.StartTransaction())
{
for (int i = 0; i < keys; i++)
{
var key = new byte[rnd.Next(10, 50)];
rnd.NextBytes(key);
var value = new byte[rnd.Next(50, 500)];
rnd.NextBytes(value);
tr.CreateOrUpdateKeyValueUnsafe(key, value);
}
tr.Commit();
}
}
using (IKeyValueDB db = CreateKeyValueDB(fileCollection))
{
}
}
}
void CreateRandomKeySequence(int keys)
{
var sw = Stopwatch.StartNew();
var rnd = new Random(1234);
using (var fileCollection = CreateTestFileCollection())
{
using (IKeyValueDB db = CreateKeyValueDB(fileCollection, new NoCompressionStrategy()))
{
db.DurableTransactions = true;
using (var tr = db.StartTransaction())
{
for (int i = 0; i < keys; i++)
{
var key = new byte[rnd.Next(10, 50)];
rnd.NextBytes(key);
var value = new byte[rnd.Next(50, 500)];
rnd.NextBytes(value);
tr.CreateOrUpdateKeyValueUnsafe(key, value);
}
tr.Commit();
}
}
}
Console.WriteLine("CreateSequence:" + sw.Elapsed.TotalMilliseconds);
}
public void Open(IKeyValueDB keyValueDB, bool dispose)
{
if (keyValueDB == null)
{
throw new ArgumentNullException(nameof(keyValueDB));
}
_keyValueDB = keyValueDB;
_dispose = dispose;
_tableInfoResolver = new TableInfoResolver(keyValueDB, this);
_tablesInfo = new TablesInfo(_tableInfoResolver);
_relationsInfoResolver = new RelationInfoResolver(this);
_relationsInfo = new RelationsInfo(_relationsInfoResolver);
_lastObjId = 0;
using (var tr = _keyValueDB.StartTransaction())
{
tr.SetKeyPrefix(AllObjectsPrefix);
if (tr.FindLastKey())
{
_lastObjId = (long)new KeyValueDBKeyReader(tr).ReadVUInt64();
}
_tablesInfo.LoadTables(LoadTablesEnum(tr));
_relationsInfo.LoadRelations(LoadRelationNamesEnum(tr));
tr.SetKeyPrefix(null);
if (tr.FindExactKey(LastDictIdKey))
{
_lastDictId = new ByteArrayReader(tr.GetValueAsByteArray()).ReadVUInt64();
}
}
}
void OpenDBSpeedTest()
{
GC.Collect();
GC.WaitForPendingFinalizers();
GC.Collect();
var memStart = GC.GetTotalMemory(false);
_sw.Start();
using (var fileCollection = OpenTestFileCollection())
{
using (IKeyValueDB db = CreateKeyValueDB(fileCollection))
{
_sw.Stop();
GC.Collect();
GC.WaitForPendingFinalizers();
GC.Collect();
var memFinish = GC.GetTotalMemory(false);
Console.WriteLine("Time to open DB : {0,15}ms Memory: {1}KB {2}KB", _sw.Elapsed.TotalMilliseconds,
(memFinish - memStart) / 1024, Process.GetCurrentProcess().WorkingSet64 / 1024);
Console.WriteLine(db.CalcStats());
_sw.Restart();
}
}
_sw.Stop();
Console.WriteLine("Time to close DB: {0,15}ms", _sw.Elapsed.TotalMilliseconds);
}
void CheckKeySequence(int keys)
{
var sw = Stopwatch.StartNew();
var key = new byte[8];
using (var fileCollection = CreateTestFileCollection())
{
using (IKeyValueDB db = CreateKeyValueDB(fileCollection, new NoCompressionStrategy()))
{
using (var tr = db.StartTransaction())
{
for (int i = 0; i < keys; i++)
{
int o = 0;
PackUnpack.PackVUInt(key, ref o, (uint)i);
tr.Find(ByteBuffer.NewSync(key, 0, o));
}
tr.Commit();
}
}
}
Console.WriteLine("CheckSequence:" + sw.Elapsed.TotalMilliseconds);
}
void ReadAllValues(IKeyValueDB kv)
{
using var tr = kv.StartTransaction();
tr.InvalidateCurrentKey();
while (tr.FindNextKey(ReadOnlySpan <byte> .Empty))
{
tr.GetValue();
}
}
public ObjectDbTableIOCTest()
{
_lowDb = new InMemoryKeyValueDB();
var builder = new ContainerBuilder();
builder.RegisterInstance <string>("Hello").Named <string>("param");
builder.RegisterType <Item>().AsSelf();
_container = builder.Build();
OpenDb();
}
protected ObjectDbTestBase(ITestOutputHelper output)
{
_output = output;
_lowDb = new BTreeKeyValueDB(new KeyValueDBOptions()
{
CompactorScheduler = null,
Compression = new NoCompressionStrategy(),
FileCollection = new InMemoryFileCollection()
});
OpenDb();
}
protected void ReopenEmptyDb()
{
_db.Dispose();
_lowDb.Dispose();
_lowDb = new BTreeKeyValueDB(new KeyValueDBOptions()
{
CompactorScheduler = null,
Compression = new NoCompressionStrategy(),
FileCollection = new InMemoryFileCollection()
});
OpenDb();
}
void ReadAllValues(IKeyValueDB kv)
{
using (var tr = kv.StartTransaction())
{
if (tr.FindFirstKey())
{
do
{
tr.GetValueAsByteArray();
} while (tr.FindNextKey());
}
}
}
public void Open(IKeyValueDB keyValueDB, bool dispose)
{
if (keyValueDB == null) throw new ArgumentNullException("keyValueDB");
_keyValueDB = keyValueDB;
_dispose = dispose;
_tableInfoResolver = new TableInfoResolver(keyValueDB, this);
_tablesInfo = new TablesInfo(_tableInfoResolver);
_lastObjId = 0;
using (var tr = _keyValueDB.StartTransaction())
{
tr.SetKeyPrefix(AllObjectsPrefix);
if (tr.FindLastKey())
{
_lastObjId = (long)new KeyValueDBKeyReader(tr).ReadVUInt64();
}
_tablesInfo.LoadTables(LoadTablesEnum(tr));
}
}
public void Open(IKeyValueDB keyValueDB, bool dispose, DBOptions options)
{
if (keyValueDB == null)
{
throw new ArgumentNullException(nameof(keyValueDB));
}
_keyValueDB = keyValueDB;
_dispose = dispose;
_type2Name = options.CustomType2NameRegistry ?? new Type2NameRegistry();
_polymorphicTypesRegistry = new PolymorphicTypesRegistry();
AutoRegisterTypes = options.AutoRegisterType;
ActualOptions = options;
_symmetricCipher = options.SymmetricCipher ?? new InvalidSymmetricCipher();
_tableInfoResolver = new TableInfoResolver(keyValueDB, this);
_tablesInfo = new TablesInfo(_tableInfoResolver);
_relationsInfoResolver = new RelationInfoResolver(this);
_relationsInfo = new RelationsInfo(_relationsInfoResolver);
using (var tr = _keyValueDB.StartTransaction())
{
_lastObjId = (long)tr.GetUlong(0);
_lastDictId = tr.GetUlong(1);
if (_lastObjId == 0)
{
tr.SetKeyPrefix(AllObjectsPrefix);
if (tr.FindLastKey())
{
_lastObjId = (long)new KeyValueDBKeyReader(tr).ReadVUInt64();
}
}
_tablesInfo.LoadTables(LoadTablesEnum(tr));
_relationsInfo.LoadRelations(LoadRelationNamesEnum(tr));
if (_lastDictId == 0)
{
tr.SetKeyPrefix(null);
if (tr.FindExactKey(LastDictIdKey))
{
_lastDictId = new ByteArrayReader(tr.GetValueAsByteArray()).ReadVUInt64();
}
}
}
}
public PersistentBuildCache(string dir)
{
var cacheIndex = 0;
while (cacheIndex < 100)
{
_mutex = new Mutex(false, @"Global\bbcoreCache" + cacheIndex);
if (_mutex.WaitOne(10))
{
break;
}
_mutex.Dispose();
_mutex = null;
cacheIndex++;
}
if (_mutex == null)
{
return;
}
_dir = dir + "/cache" + (cacheIndex == 0 ? "" : cacheIndex.ToString());
if (!new DirectoryInfo(_dir).Exists)
{
Directory.CreateDirectory(_dir);
}
_diskFileCollection = new OnDiskFileCollection(_dir);
_kvdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = _diskFileCollection,
Compression = new SnappyCompressionStrategy(),
FileSplitSize = 100000000
});
_odb = new ObjectDB();
_odb.Open(_kvdb, false);
using (var tr = _odb.StartWritingTransaction().Result)
{
_tsConfiguration = tr.InitRelation <ITSConfigurationTable>("tsconf");
_tsRelation = tr.InitRelation <ITSFileBuildCacheTable>("ts");
_tsHashedContent = tr.InitRelation <IHashedContentTable>("hashedContent");
tr.Commit();
}
}
void CheckDBTest(int keys)
{
var rnd = new Random(1234);
using (var fileCollection = OpenTestFileCollection())
{
using (IKeyValueDB db = CreateKeyValueDB(fileCollection))
{
using (var tr = db.StartTransaction())
{
if (tr.GetKeyValueCount() != keys)
{
throw new Exception("KeyCount does not match");
}
for (var i = 0; i < keys; i++)
{
var key = new byte[rnd.Next(10, 50)];
rnd.NextBytes(key);
var value = new byte[rnd.Next(50, 500)];
rnd.NextBytes(value);
if (!tr.FindExactKey(key))
{
throw new Exception("Key not found");
}
var value2 = tr.GetValueAsByteArray();
if (value.Length != value2.Length)
{
throw new Exception("value length different");
}
for (var j = 0; j < value.Length; j++)
{
if (value[j] != value2[j])
{
throw new Exception("value different");
}
}
}
}
}
}
}
void DoWork5(bool alsoDoReads)
{
_sw.Restart();
long pureDataLength = 0;
using (var fileCollection = CreateTestFileCollection())
using (IKeyValueDB db = CreateKeyValueDB(fileCollection))
{
for (int i = 0; i < 200; i++)
{
long pureDataLengthPrevTr = pureDataLength;
using (var tr = db.StartTransaction())
{
for (int j = 0; j < 200; j++)
{
tr.CreateOrUpdateKeyValue(ByteBuffer.NewAsync(new[] { (byte)j, (byte)i }), ByteBuffer.NewAsync(new byte[1 + i * j]));
pureDataLength += 2 + 1 + i * j;
}
if (alsoDoReads)
{
using (var trCheck = db.StartTransaction())
{
long pureDataLengthCheck = 0;
while (trCheck.FindNextKey())
{
pureDataLengthCheck += trCheck.GetKey().Length + trCheck.GetValue().Length;
}
if (pureDataLengthCheck != pureDataLengthPrevTr)
{
throw new Exception("Transactions are not in serializable mode");
}
}
}
tr.Commit();
}
}
}
_sw.Stop();
Console.WriteLine("Pure data length: {0}", pureDataLength);
Console.WriteLine("Time: {0,15}ms", _sw.Elapsed.TotalMilliseconds);
}
public void Open(IKeyValueDB keyValueDB, bool dispose)
{
if (keyValueDB == null)
{
throw new ArgumentNullException("keyValueDB");
}
_keyValueDB = keyValueDB;
_dispose = dispose;
_tableInfoResolver = new TableInfoResolver(keyValueDB, this);
_tablesInfo = new TablesInfo(_tableInfoResolver);
_lastObjId = 0;
using (var tr = _keyValueDB.StartTransaction())
{
tr.SetKeyPrefix(AllObjectsPrefix);
if (tr.FindLastKey())
{
_lastObjId = (long)new KeyValueDBKeyReader(tr).ReadVUInt64();
}
_tablesInfo.LoadTables(LoadTablesEnum(tr));
}
}
void Initialize()
{
_dbdir = Path.GetTempPath() + "/deleteMeDB";
Directory.Delete(_dbdir, true);
Directory.CreateDirectory(_dbdir);
_fc = new OnDiskFileCollection(_dbdir);
_kvdb = new KeyValueDB(new KeyValueDBOptions
{
Compression = new SnappyCompressionStrategy(),
FileSplitSize = 100 * 1024 * 1024,
FileCollection = _fc
});
_kvdb.Logger = this;
_odb = new ObjectDB();
_odb.Open(_kvdb, false);
using (var tr = _odb.StartWritingTransaction().GetAwaiter().GetResult())
{
_personRelation = tr.InitRelation <IPersonTable>("Person");
tr.Commit();
}
}
public PersistentBuildCache(string dir)
{
_dir = dir + "/cache";
if (!new DirectoryInfo(_dir).Exists)
{
Directory.CreateDirectory(_dir);
}
_diskFileCollection = new OnDiskFileCollection(_dir);
_kvdb = new KeyValueDB(new KeyValueDBOptions {
FileCollection = _diskFileCollection,
Compression = new SnappyCompressionStrategy(),
FileSplitSize = 100000000
});
_odb = new ObjectDB();
_odb.Open(_kvdb, false);
using (var tr = _odb.StartWritingTransaction().Result)
{
_tsConfiguration = tr.InitRelation <ITSConfigurationTable>("tsconf");
_tsRelation = tr.InitRelation <ITSFileBuildCacheTable>("ts");
tr.Commit();
}
}
public FindUnusedKeysVisitor()
{
_memoryFileCollection = new InMemoryFileCollection();
_keyValueDb = new KeyValueDB(_memoryFileCollection);
}
public void HugeTest()
{
const int keyCount = 100;
using (var fileCollection = CreateTestFileCollection())
{
_sw.Start();
using (IKeyValueDB db = CreateKeyValueDB(fileCollection, new NoCompressionStrategy()))
{
var key = new byte[100];
var value = new byte[100000000];
for (int i = 0; i < keyCount; i++)
{
using (var tr = db.StartTransaction())
{
key[0] = (byte)(i / 100);
key[1] = (byte)(i % 100);
value[100] = (byte)(i / 100);
value[200] = (byte)(i % 100);
tr.CreateOrUpdateKeyValue(key, value);
tr.Commit();
}
}
}
_sw.Stop();
Console.WriteLine("Time to create 10GB DB: {0,15}ms", _sw.Elapsed.TotalMilliseconds);
_sw.Restart();
using (IKeyValueDB db = new KeyValueDB(fileCollection))
{
_sw.Stop();
Console.WriteLine("Time to open 10GB DB: {0,15}ms", _sw.Elapsed.TotalMilliseconds);
_sw.Restart();
var key = new byte[100];
for (int i = 0; i < keyCount; i++)
{
using (var tr = db.StartTransaction())
{
key[0] = (byte)(i / 100);
key[1] = (byte)(i % 100);
tr.FindExactKey(key);
var value = tr.GetValueAsByteArray();
if (value[100] != (byte)(i / 100))
{
throw new InvalidDataException();
}
if (value[200] != (byte)(i % 100))
{
throw new InvalidDataException();
}
}
}
_sw.Stop();
Console.WriteLine("Time to read all values 10GB DB: {0,15}ms", _sw.Elapsed.TotalMilliseconds);
Console.WriteLine(db.CalcStats());
}
_sw.Restart();
using (IKeyValueDB db = CreateKeyValueDB(fileCollection))
{
_sw.Stop();
Console.WriteLine("Time to open2 10GB DB: {0,15}ms", _sw.Elapsed.TotalMilliseconds);
_sw.Restart();
var key = new byte[100];
for (int i = 0; i < keyCount; i++)
{
using (var tr = db.StartTransaction())
{
key[0] = (byte)(i / 100);
key[1] = (byte)(i % 100);
tr.FindExactKey(key);
var value = tr.GetValueAsByteArray();
if (value[100] != (byte)(i / 100))
{
throw new InvalidDataException();
}
if (value[200] != (byte)(i % 100))
{
throw new InvalidDataException();
}
}
}
_sw.Stop();
Console.WriteLine("Time to read2 all values 10GB DB: {0,15}ms", _sw.Elapsed.TotalMilliseconds);
Console.WriteLine(db.CalcStats());
}
}
}
public void Open(IKeyValueDB keyValueDB, bool dispose)
{
if (keyValueDB == null) throw new ArgumentNullException(nameof(keyValueDB));
_keyValueDB = keyValueDB;
_dispose = dispose;
_tableInfoResolver = new TableInfoResolver(keyValueDB, this);
_tablesInfo = new TablesInfo(_tableInfoResolver);
_relationsInfoResolver = new RelationInfoResolver(this);
_relationsInfo = new RelationsInfo(_relationsInfoResolver);
_lastObjId = 0;
using (var tr = _keyValueDB.StartTransaction())
{
tr.SetKeyPrefix(AllObjectsPrefix);
if (tr.FindLastKey())
{
_lastObjId = (long)new KeyValueDBKeyReader(tr).ReadVUInt64();
}
_tablesInfo.LoadTables(LoadTablesEnum(tr));
_relationsInfo.LoadRelations(LoadRelationNamesEnum(tr));
tr.SetKeyPrefix(null);
if (tr.FindExactKey(LastDictIdKey))
{
_lastDictId = new ByteArrayReader(tr.GetValueAsByteArray()).ReadVUInt64();
}
}
}
internal TableInfoResolver(IKeyValueDB keyValueDB, ObjectDB objectDB)
{
_keyValueDB = keyValueDB;
_objectDB = objectDB;
}
public ObjectDbTableFreeContentTest()
{
_lowDb = new InMemoryKeyValueDB();
OpenDb();
}
public void Setup()
{
_lowDb = new InMemoryKeyValueDB();
OpenDb();
}
public FindUnusedKeysVisitor()
{
_memoryFileCollection = new InMemoryFileCollection();
_keyValueDb = new KeyValueDB(_memoryFileCollection);
}
public ObjectDbTableUpgradeTest()
{
_lowDb = new InMemoryKeyValueDB();
OpenDb();
}
public InMemoryKeyValueDBWithCount()
{
_keyValueDB = new InMemoryKeyValueDB();
}
public ODBIteratorTest()
{
_lowDb = new InMemoryKeyValueDB();
OpenDb();
}
public void Dispose()
{
_keyValueDb?.Dispose();
_keyValueDb = null;
_memoryFileCollection?.Dispose();
_memoryFileCollection = null;
}
public ObjectDbTableUpgradeTest()
{
_lowDb = new InMemoryKeyValueDB();
OpenDb();
}
public ObjectDbTableRemoveOptimizeTest()
{
_lowDb = new InMemoryKeyValueDBWithCount();
OpenDb();
}
public ObjectDbTableFreeContentTest()
{
_lowDb = new InMemoryKeyValueDB();
OpenDb();
}
public ODBIteratorTest()
{
_lowDb = new InMemoryKeyValueDB();
OpenDb();
}
internal TableInfoResolver(IKeyValueDB keyValueDB, ObjectDB objectDB)
{
_keyValueDB = keyValueDB;
_objectDB = objectDB;
}
void OpenDB()
{
_lowDB = new KeyValueDB();
_lowDB.Open(_dbstream, false);
_db = new ObjectDB();
_db.Open(_lowDB, true);
}
public KeyValueDBReplayProxy(IKeyValueDB db, AbstractBufferedWriter log)
{
_db = db;
_log = log;
}
public void Open(IKeyValueDB keyValueDB, bool dispose)
{
Open(keyValueDB, dispose, new DBOptions());
}
public FindUnusedKeysTest()
{
_lowDb = new InMemoryKeyValueDB();
OpenDb();
}
