public void Open(IKeyValueDB keyValueDB, bool dispose, DBOptions options) { _keyValueDB = keyValueDB ?? throw new ArgumentNullException(nameof(keyValueDB)); _dispose = dispose; _type2Name = options.CustomType2NameRegistry ?? new Type2NameRegistry(); _polymorphicTypesRegistry = new PolymorphicTypesRegistry(); AutoRegisterTypes = options.AutoRegisterType; ActualOptions = options; SymmetricCipher = options.SymmetricCipher ?? new InvalidSymmetricCipher(); _tableInfoResolver = new TableInfoResolver(keyValueDB, this); _tablesInfo = new TablesInfo(_tableInfoResolver); _relationsInfoResolver = new RelationInfoResolver(this); _relationsInfo = new RelationsInfo(_relationsInfoResolver); using var tr = _keyValueDB.StartTransaction(); _lastObjId = (long)tr.GetUlong(0); _lastDictId = tr.GetUlong(1); if (_lastObjId == 0) { if (tr.FindLastKey(AllObjectsPrefix)) { _lastObjId = (long)PackUnpack.UnpackVUInt(tr.GetKey().Slice(AllObjectsPrefixLen)); } } _tablesInfo.LoadTables(LoadTablesEnum(tr)); _relationsInfo.LoadRelations(LoadRelationNamesEnum(tr)); if (_lastDictId == 0) { if (tr.FindExactKey(LastDictIdKey)) { _lastDictId = PackUnpack.UnpackVUInt(tr.GetValue()); } } }
void OpenDB() { _lowDB = new KeyValueDB(); _lowDB.Open(_dbstream, false); _db = new ObjectDB(); _db.Open(_lowDB, true); }
public void Dispose() { _keyValueDb?.Dispose(); _keyValueDb = null; _memoryFileCollection?.Dispose(); _memoryFileCollection = null; }
static Task StartReader(IKeyValueDB lowDb, string id, int sleep, CancellationToken cancellationToken) { return(Task.Run(async() => { while (true) { var transaction = lowDb.StartReadOnlyTransaction(); transaction.FindFirstKey(); var keyValueCount = transaction.GetKeyValueCount(); for (long kv = 0; kv < keyValueCount; kv++) { transaction.GetKey(); transaction.GetValue(); transaction.FindNextKey(); await Task.Delay(sleep, cancellationToken); if (cancellationToken.IsCancellationRequested) { return; } } Console.Write($"{id} "); } }, cancellationToken)); }
void CreateTestDB(int keys) { var rnd = new Random(1234); using (var fileCollection = CreateTestFileCollection()) { using (IKeyValueDB db = CreateKeyValueDB(fileCollection)) { using (var tr = db.StartTransaction()) { for (int i = 0; i < keys; i++) { var key = new byte[rnd.Next(10, 50)]; rnd.NextBytes(key); var value = new byte[rnd.Next(50, 500)]; rnd.NextBytes(value); tr.CreateOrUpdateKeyValueUnsafe(key, value); } tr.Commit(); } } using (IKeyValueDB db = CreateKeyValueDB(fileCollection)) { } } }
void CreateRandomKeySequence(int keys) { var sw = Stopwatch.StartNew(); var rnd = new Random(1234); using (var fileCollection = CreateTestFileCollection()) { using (IKeyValueDB db = CreateKeyValueDB(fileCollection, new NoCompressionStrategy())) { db.DurableTransactions = true; using (var tr = db.StartTransaction()) { for (int i = 0; i < keys; i++) { var key = new byte[rnd.Next(10, 50)]; rnd.NextBytes(key); var value = new byte[rnd.Next(50, 500)]; rnd.NextBytes(value); tr.CreateOrUpdateKeyValueUnsafe(key, value); } tr.Commit(); } } } Console.WriteLine("CreateSequence:" + sw.Elapsed.TotalMilliseconds); }
public void Open(IKeyValueDB keyValueDB, bool dispose) { if (keyValueDB == null) { throw new ArgumentNullException(nameof(keyValueDB)); } _keyValueDB = keyValueDB; _dispose = dispose; _tableInfoResolver = new TableInfoResolver(keyValueDB, this); _tablesInfo = new TablesInfo(_tableInfoResolver); _relationsInfoResolver = new RelationInfoResolver(this); _relationsInfo = new RelationsInfo(_relationsInfoResolver); _lastObjId = 0; using (var tr = _keyValueDB.StartTransaction()) { tr.SetKeyPrefix(AllObjectsPrefix); if (tr.FindLastKey()) { _lastObjId = (long)new KeyValueDBKeyReader(tr).ReadVUInt64(); } _tablesInfo.LoadTables(LoadTablesEnum(tr)); _relationsInfo.LoadRelations(LoadRelationNamesEnum(tr)); tr.SetKeyPrefix(null); if (tr.FindExactKey(LastDictIdKey)) { _lastDictId = new ByteArrayReader(tr.GetValueAsByteArray()).ReadVUInt64(); } } }
void OpenDBSpeedTest() { GC.Collect(); GC.WaitForPendingFinalizers(); GC.Collect(); var memStart = GC.GetTotalMemory(false); _sw.Start(); using (var fileCollection = OpenTestFileCollection()) { using (IKeyValueDB db = CreateKeyValueDB(fileCollection)) { _sw.Stop(); GC.Collect(); GC.WaitForPendingFinalizers(); GC.Collect(); var memFinish = GC.GetTotalMemory(false); Console.WriteLine("Time to open DB : {0,15}ms Memory: {1}KB {2}KB", _sw.Elapsed.TotalMilliseconds, (memFinish - memStart) / 1024, Process.GetCurrentProcess().WorkingSet64 / 1024); Console.WriteLine(db.CalcStats()); _sw.Restart(); } } _sw.Stop(); Console.WriteLine("Time to close DB: {0,15}ms", _sw.Elapsed.TotalMilliseconds); }
void CheckKeySequence(int keys) { var sw = Stopwatch.StartNew(); var key = new byte[8]; using (var fileCollection = CreateTestFileCollection()) { using (IKeyValueDB db = CreateKeyValueDB(fileCollection, new NoCompressionStrategy())) { using (var tr = db.StartTransaction()) { for (int i = 0; i < keys; i++) { int o = 0; PackUnpack.PackVUInt(key, ref o, (uint)i); tr.Find(ByteBuffer.NewSync(key, 0, o)); } tr.Commit(); } } } Console.WriteLine("CheckSequence:" + sw.Elapsed.TotalMilliseconds); }
void ReadAllValues(IKeyValueDB kv) { using var tr = kv.StartTransaction(); tr.InvalidateCurrentKey(); while (tr.FindNextKey(ReadOnlySpan <byte> .Empty)) { tr.GetValue(); } }
public ObjectDbTableIOCTest() { _lowDb = new InMemoryKeyValueDB(); var builder = new ContainerBuilder(); builder.RegisterInstance <string>("Hello").Named <string>("param"); builder.RegisterType <Item>().AsSelf(); _container = builder.Build(); OpenDb(); }
protected ObjectDbTestBase(ITestOutputHelper output) { _output = output; _lowDb = new BTreeKeyValueDB(new KeyValueDBOptions() { CompactorScheduler = null, Compression = new NoCompressionStrategy(), FileCollection = new InMemoryFileCollection() }); OpenDb(); }
protected void ReopenEmptyDb() { _db.Dispose(); _lowDb.Dispose(); _lowDb = new BTreeKeyValueDB(new KeyValueDBOptions() { CompactorScheduler = null, Compression = new NoCompressionStrategy(), FileCollection = new InMemoryFileCollection() }); OpenDb(); }
void ReadAllValues(IKeyValueDB kv) { using (var tr = kv.StartTransaction()) { if (tr.FindFirstKey()) { do { tr.GetValueAsByteArray(); } while (tr.FindNextKey()); } } }
public void Open(IKeyValueDB keyValueDB, bool dispose) { if (keyValueDB == null) throw new ArgumentNullException("keyValueDB"); _keyValueDB = keyValueDB; _dispose = dispose; _tableInfoResolver = new TableInfoResolver(keyValueDB, this); _tablesInfo = new TablesInfo(_tableInfoResolver); _lastObjId = 0; using (var tr = _keyValueDB.StartTransaction()) { tr.SetKeyPrefix(AllObjectsPrefix); if (tr.FindLastKey()) { _lastObjId = (long)new KeyValueDBKeyReader(tr).ReadVUInt64(); } _tablesInfo.LoadTables(LoadTablesEnum(tr)); } }
public void Open(IKeyValueDB keyValueDB, bool dispose, DBOptions options) { if (keyValueDB == null) { throw new ArgumentNullException(nameof(keyValueDB)); } _keyValueDB = keyValueDB; _dispose = dispose; _type2Name = options.CustomType2NameRegistry ?? new Type2NameRegistry(); _polymorphicTypesRegistry = new PolymorphicTypesRegistry(); AutoRegisterTypes = options.AutoRegisterType; ActualOptions = options; _symmetricCipher = options.SymmetricCipher ?? new InvalidSymmetricCipher(); _tableInfoResolver = new TableInfoResolver(keyValueDB, this); _tablesInfo = new TablesInfo(_tableInfoResolver); _relationsInfoResolver = new RelationInfoResolver(this); _relationsInfo = new RelationsInfo(_relationsInfoResolver); using (var tr = _keyValueDB.StartTransaction()) { _lastObjId = (long)tr.GetUlong(0); _lastDictId = tr.GetUlong(1); if (_lastObjId == 0) { tr.SetKeyPrefix(AllObjectsPrefix); if (tr.FindLastKey()) { _lastObjId = (long)new KeyValueDBKeyReader(tr).ReadVUInt64(); } } _tablesInfo.LoadTables(LoadTablesEnum(tr)); _relationsInfo.LoadRelations(LoadRelationNamesEnum(tr)); if (_lastDictId == 0) { tr.SetKeyPrefix(null); if (tr.FindExactKey(LastDictIdKey)) { _lastDictId = new ByteArrayReader(tr.GetValueAsByteArray()).ReadVUInt64(); } } } }
public PersistentBuildCache(string dir) { var cacheIndex = 0; while (cacheIndex < 100) { _mutex = new Mutex(false, @"Global\bbcoreCache" + cacheIndex); if (_mutex.WaitOne(10)) { break; } _mutex.Dispose(); _mutex = null; cacheIndex++; } if (_mutex == null) { return; } _dir = dir + "/cache" + (cacheIndex == 0 ? "" : cacheIndex.ToString()); if (!new DirectoryInfo(_dir).Exists) { Directory.CreateDirectory(_dir); } _diskFileCollection = new OnDiskFileCollection(_dir); _kvdb = new KeyValueDB(new KeyValueDBOptions { FileCollection = _diskFileCollection, Compression = new SnappyCompressionStrategy(), FileSplitSize = 100000000 }); _odb = new ObjectDB(); _odb.Open(_kvdb, false); using (var tr = _odb.StartWritingTransaction().Result) { _tsConfiguration = tr.InitRelation <ITSConfigurationTable>("tsconf"); _tsRelation = tr.InitRelation <ITSFileBuildCacheTable>("ts"); _tsHashedContent = tr.InitRelation <IHashedContentTable>("hashedContent"); tr.Commit(); } }
void CheckDBTest(int keys) { var rnd = new Random(1234); using (var fileCollection = OpenTestFileCollection()) { using (IKeyValueDB db = CreateKeyValueDB(fileCollection)) { using (var tr = db.StartTransaction()) { if (tr.GetKeyValueCount() != keys) { throw new Exception("KeyCount does not match"); } for (var i = 0; i < keys; i++) { var key = new byte[rnd.Next(10, 50)]; rnd.NextBytes(key); var value = new byte[rnd.Next(50, 500)]; rnd.NextBytes(value); if (!tr.FindExactKey(key)) { throw new Exception("Key not found"); } var value2 = tr.GetValueAsByteArray(); if (value.Length != value2.Length) { throw new Exception("value length different"); } for (var j = 0; j < value.Length; j++) { if (value[j] != value2[j]) { throw new Exception("value different"); } } } } } } }
void DoWork5(bool alsoDoReads) { _sw.Restart(); long pureDataLength = 0; using (var fileCollection = CreateTestFileCollection()) using (IKeyValueDB db = CreateKeyValueDB(fileCollection)) { for (int i = 0; i < 200; i++) { long pureDataLengthPrevTr = pureDataLength; using (var tr = db.StartTransaction()) { for (int j = 0; j < 200; j++) { tr.CreateOrUpdateKeyValue(ByteBuffer.NewAsync(new[] { (byte)j, (byte)i }), ByteBuffer.NewAsync(new byte[1 + i * j])); pureDataLength += 2 + 1 + i * j; } if (alsoDoReads) { using (var trCheck = db.StartTransaction()) { long pureDataLengthCheck = 0; while (trCheck.FindNextKey()) { pureDataLengthCheck += trCheck.GetKey().Length + trCheck.GetValue().Length; } if (pureDataLengthCheck != pureDataLengthPrevTr) { throw new Exception("Transactions are not in serializable mode"); } } } tr.Commit(); } } } _sw.Stop(); Console.WriteLine("Pure data length: {0}", pureDataLength); Console.WriteLine("Time: {0,15}ms", _sw.Elapsed.TotalMilliseconds); }
public void Open(IKeyValueDB keyValueDB, bool dispose) { if (keyValueDB == null) { throw new ArgumentNullException("keyValueDB"); } _keyValueDB = keyValueDB; _dispose = dispose; _tableInfoResolver = new TableInfoResolver(keyValueDB, this); _tablesInfo = new TablesInfo(_tableInfoResolver); _lastObjId = 0; using (var tr = _keyValueDB.StartTransaction()) { tr.SetKeyPrefix(AllObjectsPrefix); if (tr.FindLastKey()) { _lastObjId = (long)new KeyValueDBKeyReader(tr).ReadVUInt64(); } _tablesInfo.LoadTables(LoadTablesEnum(tr)); } }
void Initialize() { _dbdir = Path.GetTempPath() + "/deleteMeDB"; Directory.Delete(_dbdir, true); Directory.CreateDirectory(_dbdir); _fc = new OnDiskFileCollection(_dbdir); _kvdb = new KeyValueDB(new KeyValueDBOptions { Compression = new SnappyCompressionStrategy(), FileSplitSize = 100 * 1024 * 1024, FileCollection = _fc }); _kvdb.Logger = this; _odb = new ObjectDB(); _odb.Open(_kvdb, false); using (var tr = _odb.StartWritingTransaction().GetAwaiter().GetResult()) { _personRelation = tr.InitRelation <IPersonTable>("Person"); tr.Commit(); } }
public PersistentBuildCache(string dir) { _dir = dir + "/cache"; if (!new DirectoryInfo(_dir).Exists) { Directory.CreateDirectory(_dir); } _diskFileCollection = new OnDiskFileCollection(_dir); _kvdb = new KeyValueDB(new KeyValueDBOptions { FileCollection = _diskFileCollection, Compression = new SnappyCompressionStrategy(), FileSplitSize = 100000000 }); _odb = new ObjectDB(); _odb.Open(_kvdb, false); using (var tr = _odb.StartWritingTransaction().Result) { _tsConfiguration = tr.InitRelation <ITSConfigurationTable>("tsconf"); _tsRelation = tr.InitRelation <ITSFileBuildCacheTable>("ts"); tr.Commit(); } }
public FindUnusedKeysVisitor() { _memoryFileCollection = new InMemoryFileCollection(); _keyValueDb = new KeyValueDB(_memoryFileCollection); }
public void HugeTest() { const int keyCount = 100; using (var fileCollection = CreateTestFileCollection()) { _sw.Start(); using (IKeyValueDB db = CreateKeyValueDB(fileCollection, new NoCompressionStrategy())) { var key = new byte[100]; var value = new byte[100000000]; for (int i = 0; i < keyCount; i++) { using (var tr = db.StartTransaction()) { key[0] = (byte)(i / 100); key[1] = (byte)(i % 100); value[100] = (byte)(i / 100); value[200] = (byte)(i % 100); tr.CreateOrUpdateKeyValue(key, value); tr.Commit(); } } } _sw.Stop(); Console.WriteLine("Time to create 10GB DB: {0,15}ms", _sw.Elapsed.TotalMilliseconds); _sw.Restart(); using (IKeyValueDB db = new KeyValueDB(fileCollection)) { _sw.Stop(); Console.WriteLine("Time to open 10GB DB: {0,15}ms", _sw.Elapsed.TotalMilliseconds); _sw.Restart(); var key = new byte[100]; for (int i = 0; i < keyCount; i++) { using (var tr = db.StartTransaction()) { key[0] = (byte)(i / 100); key[1] = (byte)(i % 100); tr.FindExactKey(key); var value = tr.GetValueAsByteArray(); if (value[100] != (byte)(i / 100)) { throw new InvalidDataException(); } if (value[200] != (byte)(i % 100)) { throw new InvalidDataException(); } } } _sw.Stop(); Console.WriteLine("Time to read all values 10GB DB: {0,15}ms", _sw.Elapsed.TotalMilliseconds); Console.WriteLine(db.CalcStats()); } _sw.Restart(); using (IKeyValueDB db = CreateKeyValueDB(fileCollection)) { _sw.Stop(); Console.WriteLine("Time to open2 10GB DB: {0,15}ms", _sw.Elapsed.TotalMilliseconds); _sw.Restart(); var key = new byte[100]; for (int i = 0; i < keyCount; i++) { using (var tr = db.StartTransaction()) { key[0] = (byte)(i / 100); key[1] = (byte)(i % 100); tr.FindExactKey(key); var value = tr.GetValueAsByteArray(); if (value[100] != (byte)(i / 100)) { throw new InvalidDataException(); } if (value[200] != (byte)(i % 100)) { throw new InvalidDataException(); } } } _sw.Stop(); Console.WriteLine("Time to read2 all values 10GB DB: {0,15}ms", _sw.Elapsed.TotalMilliseconds); Console.WriteLine(db.CalcStats()); } } }
public void Open(IKeyValueDB keyValueDB, bool dispose) { if (keyValueDB == null) throw new ArgumentNullException(nameof(keyValueDB)); _keyValueDB = keyValueDB; _dispose = dispose; _tableInfoResolver = new TableInfoResolver(keyValueDB, this); _tablesInfo = new TablesInfo(_tableInfoResolver); _relationsInfoResolver = new RelationInfoResolver(this); _relationsInfo = new RelationsInfo(_relationsInfoResolver); _lastObjId = 0; using (var tr = _keyValueDB.StartTransaction()) { tr.SetKeyPrefix(AllObjectsPrefix); if (tr.FindLastKey()) { _lastObjId = (long)new KeyValueDBKeyReader(tr).ReadVUInt64(); } _tablesInfo.LoadTables(LoadTablesEnum(tr)); _relationsInfo.LoadRelations(LoadRelationNamesEnum(tr)); tr.SetKeyPrefix(null); if (tr.FindExactKey(LastDictIdKey)) { _lastDictId = new ByteArrayReader(tr.GetValueAsByteArray()).ReadVUInt64(); } } }
internal TableInfoResolver(IKeyValueDB keyValueDB, ObjectDB objectDB) { _keyValueDB = keyValueDB; _objectDB = objectDB; }
public ObjectDbTableFreeContentTest() { _lowDb = new InMemoryKeyValueDB(); OpenDb(); }
public void Setup() { _lowDb = new InMemoryKeyValueDB(); OpenDb(); }
public ObjectDbTableUpgradeTest() { _lowDb = new InMemoryKeyValueDB(); OpenDb(); }
public InMemoryKeyValueDBWithCount() { _keyValueDB = new InMemoryKeyValueDB(); }
public ODBIteratorTest() { _lowDb = new InMemoryKeyValueDB(); OpenDb(); }
public ObjectDbTableRemoveOptimizeTest() { _lowDb = new InMemoryKeyValueDBWithCount(); OpenDb(); }
public KeyValueDBReplayProxy(IKeyValueDB db, AbstractBufferedWriter log) { _db = db; _log = log; }
public void Open(IKeyValueDB keyValueDB, bool dispose) { Open(keyValueDB, dispose, new DBOptions()); }
public FindUnusedKeysTest() { _lowDb = new InMemoryKeyValueDB(); OpenDb(); }