public MultiProcessTests(SharedFileSystem fileSystem) { var path = fileSystem.CreateNewDirectoryForTest(); _env = new LightningEnvironment(path); _env.Open(); }
public void DatabaseDeleteShouldRemoveAllDuplicateDataItems() { var fs = new SharedFileSystem(); using (var env = new LightningEnvironment(fs.CreateNewDirectoryForTest(), configuration: new EnvironmentConfiguration { MapSize = 1024 * 1024, MaxDatabases = 1 })) { env.Open(); using (var tx = env.BeginTransaction()) using (var db = tx.OpenDatabase(configuration: new DatabaseConfiguration() { Flags = DatabaseOpenFlags.DuplicatesSort })) { var key = "key"; var value1 = "value1"; var value2 = "value2"; tx.Put(db, key, value1); tx.Put(db, key, value2); tx.Delete(db, key); Assert.False(tx.ContainsKey(db, key)); } } }
public TransactionTests(SharedFileSystem fileSystem) { var path = fileSystem.CreateNewDirectoryForTest(); _env = new LightningEnvironment(path); _env.Open(); }
public static T WithAutogrowth <T>(this LightningEnvironment env, Func <T> action) { try { return(action()); } catch (LightningException ex) when(ex.StatusCode == LightningDB.Native.Lmdb.MDB_MAP_FULL) { if (Monitor.TryEnter(env)) { try { env.MapSize = (env.MapSize == 0 ? 1048576L : env.MapSize) << 1; return(env.WithAutogrowthRetry(action)); } finally { Monitor.Exit(env); } } else { lock (env) { return(env.WithAutogrowthRetry(action)); } } } }
public (TimeSpan openTime, long memorySize) Open() { Enviroment.Dispose(); _enviroment = null; GC.Collect(); GC.WaitForPendingFinalizers(); GC.Collect(); var memStart = GC.GetTotalMemory(true); var stopwatch = Stopwatch.StartNew(); Enviroment.Open(); using (var tx = Enviroment.BeginTransaction()) using (tx.OpenDatabase(configuration: new DatabaseConfiguration { Flags = DatabaseOpenFlags.None })) { stopwatch.Stop(); GC.Collect(); GC.WaitForPendingFinalizers(); GC.Collect(); var memFinish = GC.GetTotalMemory(false); var memDif = (memFinish - memStart); return(openTime : stopwatch.Elapsed, memorySize : memDif / 1024); } }
public void GlobalSetup() { Console.WriteLine("Global Setup Begin"); const string Path = "TestDirectory"; if (Directory.Exists(Path)) { Directory.Delete(Path, true); } Env = new LightningEnvironment(Path) { MaxDatabases = 1 }; Env.Open(); using (var tx = Env.BeginTransaction()) { DB = tx.OpenDatabase(); tx.Commit(); } RunSetup(); Console.WriteLine("Global Setup End"); }
public void Init() { Directory.CreateDirectory(_path); _env = new LightningEnvironment(_path, EnvironmentOpenFlags.None); _env.Open(); }
public LightningPersistence(LightningConfig config) { _config = config.Clone(); _cts = new CancellationTokenSource(); Console.WriteLine($"ThreadId ctor: {Thread.CurrentThread.ManagedThreadId}"); Env = new LightningEnvironment(_config.Name ?? "db") { MaxDatabases = _config.MaxTables > 0u ? (int)_config.MaxTables : 20, MapSize = (_config.StorageLimit > 0L ? (long)_config.StorageLimit : 10L) * 1024 * 1024 * 1024 }; EnvironmentOpenFlags envFlags; switch (config.SyncMode) { case LightningDbSyncMode.Fsync: envFlags = EnvironmentOpenFlags.NoThreadLocalStorage; break; case LightningDbSyncMode.Async: envFlags = EnvironmentOpenFlags.NoThreadLocalStorage | EnvironmentOpenFlags.WriteMap | EnvironmentOpenFlags.MapAsync; break; case LightningDbSyncMode.NoSync: envFlags = EnvironmentOpenFlags.NoThreadLocalStorage | EnvironmentOpenFlags.NoSync; break; case LightningDbSyncMode.ReadOnly: envFlags = EnvironmentOpenFlags.NoThreadLocalStorage | EnvironmentOpenFlags.ReadOnly; break; default: throw new ArgumentOutOfRangeException(); } Env.Open(envFlags); _envHandle = GCHandle.Alloc(Env); _tables = new ConcurrentDictionary <string, Table>(); _dupTables = new ConcurrentDictionary <string, DupTable>(); _writeQueue = new BlockingCollection <Delegates>(); var threadStart = new ThreadStart(() => { foreach (var batch in DequeueBatch(_writeQueue)) { ProcessBatch(batch); } _writeTaskCompletion.SetResult(null); }); var writeThread = new Thread(threadStart) { Name = "LMDB Writer thread", IsBackground = true }; writeThread.Start(); }
private LightningEnvironment GetLightningEnvironment() { var t0 = Stopwatch.StartNew(); var env = new LightningEnvironment(_dir) { MapSize = _mapSize, MaxDatabases = 1 }; env.Open(EnvironmentOpenFlags.NoLock); using (var tx = env.BeginTransaction()) using (tx.OpenDatabase(DatabaseName, new DatabaseConfiguration { Flags = DatabaseOpenFlags.Create })) { tx.Commit(); } t0.Stop(); if (_log.IsDebugEnabled) { _log.Debug($"Init of LMDB in directory [{_dir}] took [{t0.ElapsedMilliseconds} ms]"); } return(env); }
public void CreateEnvironmentWithAutoResize() { if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { using (var env = new LightningEnvironment(_path, new EnvironmentConfiguration { MapSize = 1048576, AutoResizeWindows = true, })) { env.Open(); } using (var env = new LightningEnvironment(_path, new EnvironmentConfiguration { MapSize = 1048576, AutoResizeWindows = true, })) { env.Open(); } using (var dbFile = File.OpenRead(Path.Combine(_path, "data.mdb"))) { Assert.Equal(8192, dbFile.Length); } } }
public void Dispose() { if (_env != null && _env.IsOpened) _env.Dispose(); _env = null; }
private LightningEnvironment NewEnvironment(out LightningDatabase db, bool delete = true) { if (delete && Directory.Exists(_path)) { Directory.Delete(_path, true); } if (!Directory.Exists(_path)) { Directory.CreateDirectory(_path); } var env = new LightningEnvironment(_path, EnvironmentOpenFlags.None) { MapSize = 1024 * 1024 * 1024 * (long)10 }; env.Open(); var tx = env.BeginTransaction(); db = tx.OpenDatabase(); tx.Commit(); return(env); }
public void Init() { Directory.CreateDirectory(_path); _env = new LightningEnvironment(_path, EnvironmentOpenFlags.None); _env.Open(); }
public static void DbWrite() { using (var env = new LightningEnvironment("db")) { env.MaxDatabases = 20; env.MapSize = 20L * 1024 * 1024 * 1024; env.Open(EnvironmentOpenFlags.NoThreadLocalStorage /*| EnvironmentOpenFlags.NoLock */); using (var db = CreateLightningDatabase(env, "custom9")) { Task runner = Task.Factory.StartNew(() => RunParallelRead(db, "outer", 10, 30), TaskCreationOptions.LongRunning); using (var tx = env.BeginTransaction()) { Console.WriteLine("Write opened"); for (int i = 0; i < 1000; i++) { tx.Put(db, Encoding.UTF8.GetBytes($"hello1 + {i}"), Encoding.UTF8.GetBytes("world")); } tx.Commit(); Console.WriteLine("Write finished"); Thread.Sleep(1000); ReadBlock(db, "main"); } runner.Wait(); } } }
public void Dispose() { if (_env != null) _env.Dispose(); _env = null; }
public void TestIterator() { using (var env = new LightningEnvironment(DbEnv)) { env.MaxDatabases = 2; env.Open(); using (var tx = env.BeginTransaction()) using (var _db = tx.OpenDatabase(DbName, new DatabaseConfiguration { Flags = DatabaseOpenFlags.Create })) //assert using (var cur = tx.CreateCursor(_db)) { while (cur.MoveNext()) { var key = ByteHelper.GetStringFromBytesASCI(cur.Current.Key); var value = ByteHelper.GetStringFromBytesASCI(cur.Current.Value); Console.WriteLine(key); } } } }
public void EnvironmentShouldBeOpened() { _env = new LightningEnvironment(_path); _env.Open(); Assert.True(_env.IsOpened); }
public void EnvironmentShouldBeOpened() { _env = new LightningEnvironment(_path); _env.Open(); Assert.True(_env.IsOpened); }
public void Dispose() { if (_env != null) { _env.Dispose(); _env = null; } }
public void EnvironmentShouldBeCreatedIfReadOnly() { _env = new LightningEnvironment(_path); _env.Open(); //readonly requires environment to have been created at least once before _env.Dispose(); _env = new LightningEnvironment(_path); _env.Open(EnvironmentOpenFlags.ReadOnly); }
public static void RunTransactionScenario(this LightningEnvironment env, Action <LightningTransaction, LightningDatabase> scenario, DatabaseOpenFlags flags = DatabaseOpenFlags.Create, TransactionBeginFlags transactionFlags = TransactionBeginFlags.None) { using var tx = env.BeginTransaction(transactionFlags); using var db = tx.OpenDatabase(configuration: new DatabaseConfiguration { Flags = flags }); scenario(tx, db); }
public LightningStore(string pathToFolder, string dbName) { _env = new LightningEnvironment(pathToFolder); _dbName = dbName; _env.MaxDatabases = 1; _env.Open(); _pool = new SpanPool <byte>(PoolSizeInBytes); }
public FinalizeTests(SharedFileSystem fileSystem) { var env = new LightningEnvironment(fileSystem.CreateNewDirectoryForTest()); env.Open(); var tx = env.BeginTransaction(); var db = tx.OpenDatabase(); tx.CreateCursor(db); }
public LightningEnvironment GetDatabase(string name) { var folder = Machine.GetServiceFolder(Id.Service, name); var db = new LightningEnvironment(folder); _simulationResources.Push(db); return(db); }
public void EnvironmentShouldBeCreatedIfReadOnly() { _env = new LightningEnvironment(_path); _env.Open(); //readonly requires environment to have been created at least once before _env.Dispose(); _env = new LightningEnvironment(_path); _env.Open(EnvironmentOpenFlags.ReadOnly); }
public LightningDbPersistence(LightningDbSettings settings) { _environment = settings.ToEnvironment(); _environment.Open(EnvironmentOpenFlags.WriteMap | EnvironmentOpenFlags.NoSync); OpenDatabase(Outgoing); OpenDatabase(Delayed); }
static LightningEnvironment BuildEnvironment(string path) { var env = new LightningEnvironment(path); env.MaxDatabases = 2; env.Open(); return(env); }
public Lightning(string envPath, int numOfDb) { env = new LightningEnvironment(envPath) { MaxDatabases = numOfDb }; env.Open(); }
public void EnvironmentShouldBeClosed() { _env = new LightningEnvironment(_path); _env.Open(); _env.Dispose(); Assert.False(_env.IsOpened); }
public void Dispose() { if (_env != null && _env.IsOpened) { _env.Dispose(); } _env = null; }
public void CanOpenEnvironmentMoreThan50Mb() { _env = new LightningEnvironment(_path) { MapSize = 55 * 1024 * 1024 }; _env.Open(); }
public void EnvironmentShouldBeClosed() { _env = new LightningEnvironment(_path); _env.Open(); _env.Dispose(); Assert.False(_env.IsOpened); }
public void EnvironmentShouldBeCreatedIfReadOnly() { //arrange //act _env = new LightningEnvironment(_path, EnvironmentOpenFlags.ReadOnly); //assert }
public void EnvironmentShouldBeCreatedIfWithoutFlags() { //arrange //act _env = new LightningEnvironment(_path, EnvironmentOpenFlags.None); //assert }
public ChangeStream(string dirPath) { _env = new LightningEnvironment(dirPath); _env.Open(); using (var tx = _env.BeginTransaction()) { _db = tx.OpenDatabase(configuration: s_defaultDbConfig); } }
public void EnvironmentShouldBeCreatedIfWithoutFlags() { //arrange //act _env = new LightningEnvironment(_path, EnvironmentOpenFlags.None); //assert }
public void EnvironmentShouldBeCreatedIfReadOnly() { //arrange //act _env = new LightningEnvironment(_path, EnvironmentOpenFlags.ReadOnly); //assert }
public FinalizeTests(SharedFileSystem fileSystem) { var env = new LightningEnvironment(fileSystem.CreateNewDirectoryForTest()); env.Open(); var tx = env.BeginTransaction(); var db = tx.OpenDatabase(); tx.CreateCursor(db); }
public void EnvironmentShouldBeCopied(bool compact) { _env = new LightningEnvironment(_path); _env.Open(); _env.CopyTo(_pathCopy, compact); if (Directory.GetFiles(_pathCopy).Length == 0) Assert.True(false, "Copied files doesn't exist"); }
public void Init() { Directory.CreateDirectory(_path); _env = new LightningEnvironment(_path, EnvironmentOpenFlags.None); _env.Open(); _txn = _env.BeginTransaction(); _db = _txn.OpenDatabase(); }
public void Init() { Directory.CreateDirectory(_path); _env = new LightningEnvironment(_path, EnvironmentOpenFlags.None); _env.Open(); _txn = _env.BeginTransaction(); _db = _txn.OpenDatabase(); }
public CursorTests(SharedFileSystem fileSystem) { var path = fileSystem.CreateNewDirectoryForTest(); _env = new LightningEnvironment(path); _env.Open(); _txn = _env.BeginTransaction(); }
public DatabaseIOTests(SharedFileSystem fileSystem) { var path = fileSystem.CreateNewDirectoryForTest(); _env = new LightningEnvironment(path); _env.MaxDatabases = 2; _env.Open(); _txn = _env.BeginTransaction(); _db = _txn.OpenDatabase(configuration: new DatabaseConfiguration {Flags = DatabaseOpenFlags.Create}); }
public void EnvironmentCreatedFromConfig() { var mapExpected = 1024*1024*20; var maxDatabaseExpected = 2; var maxReadersExpected = 3; var config = new EnvironmentConfiguration {MapSize = mapExpected, MaxDatabases = maxDatabaseExpected, MaxReaders = maxReadersExpected}; _env = new LightningEnvironment(_path, config); Assert.Equal(_env.MapSize, mapExpected); Assert.Equal(_env.MaxDatabases, maxDatabaseExpected); Assert.Equal(_env.MaxReaders, maxReadersExpected); }
public void EnvironmentShouldBeOpened() { //arrange _env = new LightningEnvironment(_path, EnvironmentOpenFlags.None); //act _env.Open(); //assert Assert.AreEqual(true, _env.IsOpened); }
internal void Configure(LightningEnvironment env) { if (_mapSize.HasValue) env.MapSize = _mapSize.Value; if (_maxDatabases.HasValue) env.MaxDatabases = _maxDatabases.Value; if (_maxReaders.HasValue) env.MaxReaders = _maxReaders.Value; }
public void RemotePut(string environment) { var env = new LightningEnvironment(environment); env.Open(); using (env) using(var tx = env.BeginTransaction()) using(var db = tx.OpenDatabase()) { tx.Put(db, Encoding.UTF8.GetBytes("hello"), Encoding.UTF8.GetBytes("world")); tx.Commit(); } }
public void MaxDatabasesWorksThroughConfigIssue62() { var config = new EnvironmentConfiguration { MaxDatabases = 2 }; _env = new LightningEnvironment(_path, config); _env.Open(); using (var tx = _env.BeginTransaction()) { tx.OpenDatabase("db1", new DatabaseConfiguration {Flags = DatabaseOpenFlags.Create}); tx.OpenDatabase("db2", new DatabaseConfiguration {Flags = DatabaseOpenFlags.Create}); tx.Commit(); } Assert.Equal(_env.MaxDatabases, 2); }
public void Cleanup() { if (_env != null && _env.IsOpened) _env.Close(); _env = null; if (Directory.Exists(_path)) Directory.Delete(_path, true); if (Directory.Exists(_pathCopy)) Directory.Delete(_pathCopy, true); }
public TransactionManager(LightningEnvironment environment, LightningTransaction parentTransaction) { if (environment == null) throw new ArgumentNullException("environment"); _environment = environment; _parentTransaction = parentTransaction; _parentHandle = parentTransaction != null ? parentTransaction._handle : IntPtr.Zero; _transactions = new ConcurrentDictionary<LightningTransaction, bool>(); _comparatorsStore = new ConcurrentDictionary<CompareFunction, bool>(); }
private static void DoTest(bool useRandomKeys = false) { var env = new LightningEnvironment(CreateNewDirectoryForTest()); env.MaxDatabases = 2; env.MapSize = 1000 * 1024 * 1025; // 100 MB env.Open(); try { DoTestImpl(env, useRandomKeys); } finally { if (env != null) env.Dispose(); } }
public MultiProcessTests(SharedFileSystem fileSystem) { var path = fileSystem.CreateNewDirectoryForTest(); _env = new LightningEnvironment(path); _env.Open(); }
public void CanOpenEnvironmentMoreThan50Mb() { _env = new LightningEnvironment(_path) { MapSize = 55 * 1024 * 1024 }; _env.Open(); }
public void EnvironmentShouldBeCreatedIfWithoutFlags() { _env = new LightningEnvironment(_path); _env.Open(); }
public void StartingTransactionBeforeEnvironmentOpen() { _env = new LightningEnvironment(_path); Assert.Throws<InvalidOperationException>(() => _env.BeginTransaction()); }
public void CanLoadAndDisposeMultipleEnvironments() { _env = new LightningEnvironment(_path); _env.Dispose(); _env = new LightningEnvironment(_path); }
public DatabaseTests(SharedFileSystem fileSystem) { var path = fileSystem.CreateNewDirectoryForTest(); _env = new LightningEnvironment(path); }
public void CanOpenEnvironmentMoreThan50Mb() { //arrange _env = new LightningEnvironment(_path, EnvironmentOpenFlags.None) { MapSize = 55 * 1024 * 1024 }; //act-assert _env.Open(); }
public void EnvironmentShouldBeCopied() { //arrange _env = new LightningEnvironment(_path, EnvironmentOpenFlags.None); _env.Open(); //act _env.CopyTo(_pathCopy); //assert if (Directory.GetFiles(_pathCopy).Length == 0) Assert.Fail("Copied files doesn't exist"); }
private static void DoTestImpl(LightningEnvironment env, bool useRandomKeys = false) { var numItemsToWrite = 1 * 1000; // One thousand //var numItemsToWrite = 10 * 1000; // Ten thousand //var numItemsToWrite = 100 * 1000; // One hundred thousand //var numItemsToWrite = 1 * 1000 * 1000; // 1 million //var numItemsToWrite = 10 * 1000 * 1000; // 10 million var randon = new Random(1773); Console.WriteLine("Using {0} keys", useRandomKeys ? "RANDOM" : "SEQUENTIAL"); var writeTimer = Stopwatch.StartNew(); // Need to specify DatabaseOpenFlags.IntegerKey if we want the items to be sorted, not entirely sure why though, // probably big/little endian related, see http://www.openldap.org/lists/openldap-bugs/201308/msg00050.html var dbConfig = new DatabaseConfiguration { Flags = DatabaseOpenFlags.Create | DatabaseOpenFlags.IntegerKey }; using (var tx = env.BeginTransaction()) using (var db = tx.OpenDatabase("test", dbConfig)) { for (var i = 0; i < numItemsToWrite; ++i) { var key = i; if (useRandomKeys) key = randon.Next(); //tx.Put(db, BitConverter.GetBytes(i), BitConverter.GetBytes(i)); var text = "Some Text plus: 'Key' = " + key.ToString("N0"); // + " " + new String('c', 1000); var data = GetBinaryEncodedString(text); tx.Put(db, BitConverter.GetBytes(key), data); } tx.Commit(); var stats = tx.GetStats(db); Console.WriteLine("Stats\n # Entries: {0:N0}\n Depth: {1}\n Page Size: {2}", stats.ms_entries.ToInt64(), stats.ms_depth, stats.ms_psize); Console.WriteLine(" Branch Pages: {0:N0}\n Leaf Pages: {1}\n Overflow Pages: {2}", stats.ms_branch_pages.ToInt64(), stats.ms_leaf_pages.ToInt64(), stats.ms_overflow_pages.ToInt64()); } writeTimer.Stop(); Console.WriteLine("Took {0,10:N2} ms ({1}) to WRITE {2,9:N0} values ({3,10:N0} WRITES/sec)", writeTimer.Elapsed.TotalMilliseconds, writeTimer.Elapsed, numItemsToWrite, numItemsToWrite / writeTimer.Elapsed.TotalMilliseconds * 1000.0); var readTimer = Stopwatch.StartNew(); var readCounter = 0; using (var tx = env.BeginTransaction()) using (var db = tx.OpenDatabase("test")) { tx.Get(db, BitConverter.GetBytes(int.MinValue)); ValueStructure currentKey = default(ValueStructure); ValueStructure currentValue = default(ValueStructure); using (var cursor = tx.CreateCursor(db)) { while (cursor.MoveNext()) { //var current = cursor.Current; cursor.GetCurrent(out currentKey, out currentValue); unsafe { var keyData = *((int*)(currentKey.data.ToPointer())); int valueLengthSize; var ptr = (byte*)currentValue.data.ToPointer(); var length = Read7BitEncodedInt(ptr, out valueLengthSize); var text = new string((sbyte*)(ptr + valueLengthSize), 0, length, Encoding.UTF8); //Console.WriteLine("{{ Key: {0:N0}, Value: \"{1}\" }}", keyData, text); } readCounter++; } } } readTimer.Stop(); Console.WriteLine("Took {0,10:N2} ms ({1}) to READ {2,9:N0} values ({3,10:N0} READS/sec)", readTimer.Elapsed.TotalMilliseconds, readTimer.Elapsed, readCounter, readCounter / readTimer.Elapsed.TotalMilliseconds * 1000.0); Console.WriteLine(); }