protected Table(WriteTransaction txn, string tableName, DatabaseOpenFlags flags) { Database = txn.Transaction.OpenDatabase(tableName, new DatabaseConfiguration { Flags = flags }); txn.Commit(); }
private LightningEnvironment NewEnvironment(out LightningDatabase db, bool delete = true) { if (delete && Directory.Exists(_path)) { Directory.Delete(_path, true); } if (!Directory.Exists(_path)) { Directory.CreateDirectory(_path); } var env = new LightningEnvironment(_path, EnvironmentOpenFlags.None) { MapSize = 1024 * 1024 * 1024 * (long)10 }; env.Open(); var tx = env.BeginTransaction(); db = tx.OpenDatabase(); tx.Commit(); return(env); }
private void DbPut(LightningTransaction tx, LightningDatabase db, string key, DurableDataEnvelope data) { var byteKey = Encoding.UTF8.GetBytes(key); var byteValue = _serializer.ToBinary(data); tx.Put(db, byteKey, byteValue); }
public static string Get(this LightningTransaction tx, LightningDatabase db, string key) { var enc = System.Text.Encoding.UTF8; var result = tx.Get(db, enc.GetBytes(key)); return(enc.GetString(result.value.CopyToNewArray())); }
public void GlobalSetup() { Console.WriteLine("Global Setup Begin"); const string Path = "TestDirectory"; if (Directory.Exists(Path)) { Directory.Delete(Path, true); } Env = new LightningEnvironment(Path) { MaxDatabases = 1 }; Env.Open(); using (var tx = Env.BeginTransaction()) { DB = tx.OpenDatabase(); tx.Commit(); } RunSetup(); Console.WriteLine("Global Setup End"); }
public void CursorShouldPutValues() { _db = _txn.OpenDatabase(configuration: new DatabaseConfiguration { Flags = DatabaseOpenFlags.Create }); PopulateCursorValues(); }
public void DropCollection(string name) { if (string.IsNullOrEmpty(name)) { throw new ArgumentException("Collection name cant not be null or empty."); } if (!CollectionExists(name)) { throw new ArgumentException("Specified collection not found in " + GetFileInfo() + " Collection = " + name); } ITransaction transaction = BeginTransactionInternal(null, TransactionBeginFlags.None); ValidateTransaction(transaction); LightningTransaction lmdbTransaction = (LightningTransaction)transaction.InnerObject; LightningDatabase dbInstance = _collectionTable[name].Collection; long dataSize = _collectionTable[name].Stats.DataSize; lmdbTransaction.DropDatabase(dbInstance); _collectionTable.Remove(name); StoreCollectionInfo(null, transaction); //StoreDocument(transaction, _metadataCollection, "collections", _collectionTable.Keys.ToArray()); ((LMDBTransaction)transaction).ChangeSize(-8192); ((LMDBTransaction)transaction).ChangeSize(-dataSize); CommitInternal(transaction); }
private LightningDatabase CreateCollectionInternal(string name, Type keyType) { ITransaction transaction = BeginTransactionInternal(null, TransactionBeginFlags.None); ValidateTransaction(transaction); LightningTransaction lmdbTransaction = (LightningTransaction)transaction.InnerObject; LightningDB.DatabaseConfiguration dbOptions = new LightningDB.DatabaseConfiguration(); dbOptions.Flags = DatabaseOpenFlags.Create; //todo: maybe add other integer types too if (keyType == typeof(long) || keyType == typeof(ulong) || keyType == typeof(int) || keyType == typeof(uint) || keyType == typeof(short) || keyType == typeof(ushort)) { dbOptions.Flags |= DatabaseOpenFlags.IntegerKey; } LightningDatabase collectionInstance = lmdbTransaction.OpenDatabase(name, dbOptions); if (name != METADATA_COLLECTION) { StoreCollectionInfo(name, transaction); } ((LMDBTransaction)transaction).ChangeSize(8192); CommitInternal(transaction); return(collectionInstance); }
public void CursorShouldBeCreated() { _db = _txn.OpenDatabase(configuration: new DatabaseConfiguration { Flags = DatabaseOpenFlags.Create }); _txn.CreateCursor(_db).Dispose(); }
public LightningCursor OpenCursor(LightningDatabase db) { var handle = CreateCursorHandle(db._handle); _cursors.TryAdd(handle, true); return new LightningCursor(db, _transaction, handle); }
public ChangeStream(string dirPath) { _env = new LightningEnvironment(dirPath); _env.Open(); using (var tx = _env.BeginTransaction()) { _db = tx.OpenDatabase(configuration: s_defaultDbConfig); } }
internal ObjectRepositoryTransaction( ObjectRepositorySettings <TKey, T> settings, LightningTransaction tx, LightningDatabase db) { _settings = settings; _tx = tx; _db = db; }
public ObjectRepository(ObjectRepositorySettings <TKey, T> settings) { _settings = settings; _env = new LightningEnvironment(settings.Path); _env.Open(); using (var tx = _env.BeginTransaction()) { _db = tx.OpenDatabase(); } }
public LMDBDataReader(IEnumerator <KeyValuePair <byte[], byte[]> > enumerator, LightningEnvironment environment, LightningDatabase db) { if (enumerator == null) { throw new ArgumentException("Enumerator can not be null."); } _enumerator = enumerator; _environment = environment; _database = db; }
public void Init() { Directory.CreateDirectory(_path); _env = new LightningEnvironment(_path, EnvironmentOpenFlags.None); _env.Open(); _txn = _env.BeginTransaction(); _db = _txn.OpenDatabase(); }
public static bool TryGet(this LightningTransaction tx, LightningDatabase db, string key, out string value) { var enc = System.Text.Encoding.UTF8; byte[] result; var found = tx.TryGet(db, enc.GetBytes(key), out result); value = enc.GetString(result); return(found); }
public void CursorShouldPutValues() { //arrange _db = _txn.OpenDatabase(options: new DatabaseOptions { Flags = DatabaseOpenFlags.None }); //act //assert this.PopulateCursorValues(); }
public void CursorShouldBeCreated() { //arrange _db = _txn.OpenDatabase(options: new DatabaseOptions { Flags = DatabaseOpenFlags.None }); //act //assert _txn.CreateCursor(_db); }
private List <PerformanceRecord> WriteInternal( string operation, IEnumerator <TestData> enumerator, long itemsPerTransaction, long numberOfTransactions, PerfTracker perfTracker, rndseq Rflag, LightningEnvironment env, LightningDatabase db) { byte[] valueToWrite = null; var records = new List <PerformanceRecord>(); var sw = new Stopwatch(); LightningDB.PutOptions putflags = LightningDB.PutOptions.None; if (Rflag == rndseq.SEQ) { putflags = LightningDB.PutOptions.AppendData; } for (var transactions = 0; transactions < numberOfTransactions; transactions++) { sw.Restart(); using (var tx = env.BeginTransaction()) { for (var i = 0; i < itemsPerTransaction; i++) { enumerator.MoveNext(); valueToWrite = GetValueToWrite(valueToWrite, enumerator.Current.ValueSize); tx.Put(db, Encoding.UTF8.GetBytes(enumerator.Current.Id.ToString("0000000000000000")), valueToWrite, putflags); } tx.Commit(); } sw.Stop(); perfTracker.Record(sw.ElapsedMilliseconds); records.Add(new PerformanceRecord { Operation = operation, Time = DateTime.Now, Duration = sw.ElapsedMilliseconds, ProcessedItems = itemsPerTransaction }); } sw.Stop(); return(records); }
public DatabaseIOTests(SharedFileSystem fileSystem) { var path = fileSystem.CreateNewDirectoryForTest(); _env = new LightningEnvironment(path); _env.MaxDatabases = 2; _env.Open(); _txn = _env.BeginTransaction(); _db = _txn.OpenDatabase(configuration: new DatabaseConfiguration {Flags = DatabaseOpenFlags.Create}); }
public static void TryDelete(this LightningTransaction tx, LightningDatabase db, byte[] key) { try { tx.Delete(db, key); } catch (LightningException ex) { if (ex.StatusCode != Lmdb.MDB_NOTFOUND) { throw; } } }
public DatabaseIOTests(SharedFileSystem fileSystem) { var path = fileSystem.CreateNewDirectoryForTest(); _env = new LightningEnvironment(path); _env.MaxDatabases = 2; _env.Open(); _txn = _env.BeginTransaction(); _db = _txn.OpenDatabase(configuration: new DatabaseConfiguration { Flags = DatabaseOpenFlags.Create }); }
private static void ReadBlock(LightningDatabase db, string name) { using (var tx = db.Environment.BeginTransaction(TransactionBeginFlags.ReadOnly)) { Console.WriteLine($"{name} - Read opened"); for (int i = 0; i < 10000000; i++) { var result = tx.Get(db, Encoding.UTF8.GetBytes("hello")); } Console.WriteLine($"{name} - Read finished"); } }
void ProcessGrantsBySubjectId(string subjectId, LightningTransaction tx, LightningDatabase mainDb, LightningDatabase indexDb, Action <LightningCursor> callback) { var subjectIdBytes = Encoding.UTF8.GetBytes(subjectId); using (var cursor = tx.CreateCursor(indexDb)) { if (cursor.MoveTo(subjectIdBytes)) { do { callback(cursor); } while (cursor.MoveNextDuplicate()); } } }
public CursorGenericEnumerable(LightningTransaction tx, LightningDatabase db) { if (tx == null) { throw new ArgumentNullException("tx"); } if (db == null) { throw new ArgumentNullException("db"); } _tx = tx; _db = db; }
private static long ReadInternal(IEnumerable <uint> ids, PerfTracker perfTracker, LightningEnvironment env, LightningDatabase db) { using (var tx = env.BeginTransaction(LightningDB.TransactionBeginFlags.ReadOnly)) using (var cursor = new LightningCursor(db, tx)) { long v = 0; foreach (var id in ids) { var value = cursor.MoveTo(Encoding.UTF8.GetBytes(id.ToString("0000000000000000"))); v += value.Value.Length; //Debug.Assert(value != null); } return(v); } }
public static void Index(LightningDatabase db, LightningTransaction tx, byte[] key, byte[] value) { // IMPORTANT: // lmdb's DUP_SORT still imposes the MaxKeySizeBytes on the length of the key + value, // so it's less ambiguous if we handle "multiple values for a single key" semantics ourselves // if (key.Length > MaxKeySizeBytes) { // FIXME: localize this var message = $"Index key length is {key.Length} but the maximum key size is {MaxKeySizeBytes}"; throw new InvalidOperationException(message); } tx.Put(db, key, value, PutOptions.NoOverwrite); }
public void CursorShouldMoveToFirst() { _db = _txn.OpenDatabase(configuration: new DatabaseConfiguration { Flags = DatabaseOpenFlags.Create }); PopulateCursorValues(); using (var cur = _txn.CreateCursor(_db)) { cur.MoveToFirst(); var lastKey = UTF8.GetString(cur.Current.Key); var lastValue = UTF8.GetString(cur.Current.Value); Assert.Equal("key1", lastKey); Assert.Equal("key1", lastValue); } }
public void ShouldIterateThroughCursorByEnumerator() { _db = _txn.OpenDatabase(configuration: new DatabaseConfiguration { Flags = DatabaseOpenFlags.Create }); PopulateCursorValues(); var i = 0; using (var cursor = _txn.CreateCursor(_db)) { foreach (var pair in cursor) { var name = "key" + ++i; Assert.Equal(name, UTF8.GetString(pair.Key)); Assert.Equal(name, UTF8.GetString(pair.Value)); } } }
public BackendServer(IEnv env, ushort port, CommitLogClient client) { _env = env; _client = client; _port = port; _le = env.GetDatabase("db"); _le.Open(EnvironmentOpenFlags.MapAsync); using (var tx = _le.BeginTransaction()) { _ld = tx.OpenDatabase(configuration: new DatabaseConfiguration() { Flags = DatabaseOpenFlags.Create }); tx.Commit(); } _store = new Store(_le, _ld); _proj = new Projection(_store); }
public static void PrintAllVulnerabilities() { if (!Environment.IsOpened) { Environment.MapSize = 2L * 1024L * 1024L * 1024L; Environment.Open(EnvironmentOpenFlags.None); } using (LightningTransaction tx = Environment.BeginTransaction(TransactionBeginFlags.ReadOnly)) { LightningDatabase db = tx.OpenDatabase(); using (LightningCursor cursor = tx.CreateCursor(db)) { foreach (KeyValuePair <byte[], byte[]> r in cursor) { string id = Encoding.UTF8.GetString(r.Key); OSSIndexVulnerability v = JsonConvert.DeserializeObject <OSSIndexVulnerability>(Encoding.UTF8.GetString(r.Value)); logger.Info("Id: {0}), Package: {1}, CVEs: {2}", id, v.Name, string.Join(" ", v.CVEs)); } } } }
public void ShouldMoveNextMultiple() { _db = _txn.OpenDatabase(configuration: new DatabaseConfiguration { Flags = DatabaseOpenFlags.DuplicatesFixed | DatabaseOpenFlags.Create }); var original = new[] { 1, 2, 3, 4, 5 }; var originalBytes = original.Select(BitConverter.GetBytes).ToArray(); using (var cur = _txn.CreateCursor(_db)) cur.PutMultiple(UTF8.GetBytes("TestKey"), originalBytes); using (var cur = _txn.CreateCursor(_db)) { cur.MoveNextMultiple(); var resultPair = cur.Current; Assert.Equal("TestKey", UTF8.GetString(resultPair.Key)); var result = resultPair.Value.Split(sizeof(int)) .Select(x => BitConverter.ToInt32(x.ToArray(), 0)).ToArray(); Assert.Equal(original, result); } }
public void CursorShouldDeleteElements() { _db = _txn.OpenDatabase(configuration: new DatabaseConfiguration { Flags = DatabaseOpenFlags.Create }); PopulateCursorValues(); using (var cursor = _txn.CreateCursor(_db)) { for (int i = 0; i < 2; ++i) { cursor.MoveNext(); cursor.Delete(); } } using (var cursor = _txn.CreateCursor(_db)) { var foundDeleted = cursor.Select(x => UTF8.GetString(x.Key)) .Any(x => x == "key1" || x == "key2"); Assert.False(foundDeleted); } }
public static bool PutVulnerabilities(List <OSSIndexVulnerability> vulnerabilities) { if (!Environment.IsOpened) { Environment.MapSize = 2L * 1024L * 1024L * 1024L; Environment.Open(EnvironmentOpenFlags.None); } using (LightningTransaction tx = Environment.BeginTransaction()) using (LightningDatabase db = tx.OpenDatabase(null, new DatabaseConfiguration { Flags = DatabaseOpenFlags.Create })) { try { foreach (OSSIndexVulnerability v in vulnerabilities) { /* * using (MemoryStream ms = new MemoryStream()) * using (BsonWriter writer = new BsonWriter(ms)) * { * JsonSerializer serializer = new JsonSerializer(); * serializer.Serialize(writer, v); * } */ tx.Put(db, Encoding.UTF8.GetBytes(string.IsNullOrEmpty(v.Vid) ? v.Url : v.Vid + "#" + v.Group), Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(v))); } tx.Commit(); logger.Debug("Committed {0} vulnerabilities to database.", vulnerabilities.Count); return(true); } catch (LightningException e) { logger.Error("Exception thrown attempting to write vulnerabilities to database."); logger.Error(e); return(false); } } }
public void ShouldPutMultiple() { //arrange _db = _txn.OpenDatabase(options: new DatabaseOptions { Flags = DatabaseOpenFlags.DuplicatesFixed }); var values = new[] { 1, 2, 3, 4, 5 }; using (var cur = _txn.CreateCursor(_db)) cur.PutMultiple("TestKey", values); var pairs = new KeyValuePair<string, int>[values.Length]; //act using (var cur = _txn.CreateCursor(_db)) { for (var i = 0; i < values.Length; i++) cur.MoveNextDuplicate<string, int>(out pairs[i]); } //assert for (var i = 0; i < values.Length; i++) Assert.AreEqual(values[i], pairs[i].Value); }
public void ShouldIterateThroughCursor() { _db = _txn.OpenDatabase(configuration: new DatabaseConfiguration { Flags = DatabaseOpenFlags.Create }); PopulateCursorValues(); using (var cur = _txn.CreateCursor(_db)) { var i = 0; while (cur.MoveNext()) { var key = UTF8.GetString(cur.Current.Key); var value = UTF8.GetString(cur.Current.Value); var name = "key" + ++i; Assert.Equal(name, key); Assert.Equal(name, value); } Assert.NotEqual(0, i); } }
public void CursorShouldBeCreated() { _db = _txn.OpenDatabase(configuration: new DatabaseConfiguration { Flags = DatabaseOpenFlags.Create }); _txn.CreateCursor(_db); }
public bool IsReleased(LightningDatabase db) { return !_databasesForReuse.Contains(db._handle); }
public void ShouldGetMultiple() { //arrange _db = _txn.OpenDatabase(options: new DatabaseOptions { Flags = DatabaseOpenFlags.DuplicatesFixed }); var values = new[] { 1, 2, 3, 4, 5 }; using (var cur = _txn.CreateCursor(_db)) cur.PutMultiple("TestKey", values); bool result; int[] resultArray; using (var cur = _txn.CreateCursor(_db)) { KeyValuePair<string, int> pair; cur.MoveNext(out pair); result = cur.GetMultiple(out resultArray); } Assert.IsTrue(result); CollectionAssert.AreEqual(values, resultArray); }
public void Close(LightningDatabase db, bool releaseHandle) { lock (_syncObject) { if (IsReleased(db)) return; Reuse(db); if (!releaseHandle) return; try { NativeMethods.Library.mdb_dbi_close(_environmentHandle, db._handle); } finally { Release(db); } } }
private void Reuse(LightningDatabase db) { DatabaseHandleCacheEntry entry; _openedDatabases.TryRemove(db.Name, out entry); }
private void Release(LightningDatabase db) { _databasesForReuse.Remove(db._handle); }
public bool IsOpen(LightningDatabase db) { DatabaseHandleCacheEntry entry; return _openedDatabases.TryGetValue(db.Name, out entry) && entry.Handle == db._handle; }
public void CursorShouldMoveToLast() { //arrange _db = _txn.OpenDatabase(options: new DatabaseOptions { Flags = DatabaseOpenFlags.None }); this.PopulateCursorValues(); //assert using (var cur = _txn.CreateCursor(_db)) { var last = cur.MoveToLast().Value; var lastKey = Encoding.UTF8.GetString(last.Key); var lastValue = Encoding.UTF8.GetString(last.Value); Assert.AreEqual("key5", lastKey); Assert.AreEqual("key5", lastValue); } }
public void CursorShouldDeleteElements() { //arrange _db = _txn.OpenDatabase(options: new DatabaseOptions { Flags = DatabaseOpenFlags.None }); this.PopulateCursorValues(); using (var cur = _txn.CreateCursor(_db)) { var i = 0; //act for (var current = cur.MoveToFirstBy(); current.PairExists && i < 2; current = cur.MoveNextBy(), i++) { cur.Delete(); } //assert for (var current = cur.MoveToFirstBy(); current.PairExists; current = cur.MoveNextBy()) { var key = current.Key<string>(); Assert.AreNotEqual("key1", key); Assert.AreNotEqual("key2", key); } } }
public void ShouldMoveNextMultiple() { //arrange _db = _txn.OpenDatabase(options: new DatabaseOptions { Flags = DatabaseOpenFlags.DuplicatesFixed }); const string key = "TestKey"; var values = new[] { 1, 2, 3, 4, 5 }; using (var cur = _txn.CreateCursor(_db)) cur.PutMultiple(key, values); bool result; KeyValuePair<string, int[]> resultPair; using (var cur = _txn.CreateCursor(_db)) result = cur.MoveNextMultiple(out resultPair); Assert.IsTrue(result); Assert.AreEqual(key, resultPair.Key); CollectionAssert.AreEqual(values, resultPair.Value); }
public void CursorShouldDeleteElements() { _db = _txn.OpenDatabase(configuration: new DatabaseConfiguration { Flags = DatabaseOpenFlags.Create }); PopulateCursorValues(); using (var cursor = _txn.CreateCursor(_db)) { for (int i = 0; i < 2; ++i) { cursor.MoveNext(); cursor.Delete(); } } using(var cursor = _txn.CreateCursor(_db)) { var foundDeleted = cursor.Select(x => UTF8.GetString(x.Key)) .Any(x => x == "key1" || x == "key2"); Assert.False(foundDeleted); } }
public void ShouldIterateThroughCursorByExtensions() { //arrange _db = _txn.OpenDatabase(options: new DatabaseOptions { Flags = DatabaseOpenFlags.None }); this.PopulateCursorValues(); using (var cur = _txn.CreateCursor(_db)) { var i = 0; //act for(var current = cur.MoveToFirstBy(); current.PairExists; current = cur.MoveNextBy()) { var pair = current.Pair<string, string>(); var name = "key" + ++i; //assert Assert.AreEqual(name, pair.Key); Assert.AreEqual(name, pair.Value); } } }
public void ShouldIterateThroughCursorByEnumerator() { //arrange _db = _txn.OpenDatabase(options: new DatabaseOptions { Flags = DatabaseOpenFlags.None }); this.PopulateCursorValues(); var i = 0; foreach (var pair in _txn.EnumerateDatabase<string, string>(_db)) { var name = "key" + ++i; //assert Assert.AreEqual(name, pair.Key); Assert.AreEqual(name, pair.Value); } }
public void ShouldIterateThroughCursorByEnumerator() { _db = _txn.OpenDatabase(configuration: new DatabaseConfiguration { Flags = DatabaseOpenFlags.Create }); PopulateCursorValues(); var i = 0; using(var cursor = _txn.CreateCursor(_db)) { foreach (var pair in cursor) { var name = "key" + ++i; Assert.Equal(name, UTF8.GetString(pair.Key)); Assert.Equal(name, UTF8.GetString(pair.Value)); } } }
public void ShouldIterateThroughCursor() { //arrange _db = _txn.OpenDatabase(options: new DatabaseOptions { Flags = DatabaseOpenFlags.None }); this.PopulateCursorValues(); using (var cur = _txn.CreateCursor(_db)) { var i = 0; //act while (true) { var current = cur.MoveNext(); if (!current.HasValue) break; var key = Encoding.UTF8.GetString(current.Value.Key); var value = Encoding.UTF8.GetString(current.Value.Value); var name = "key" + ++i; //assert Assert.AreEqual(name, key); Assert.AreEqual(name, value); } } }
public void ShouldPutMultiple() { _db = _txn.OpenDatabase(configuration: new DatabaseConfiguration { Flags = DatabaseOpenFlags.DuplicatesFixed | DatabaseOpenFlags.Create }); var values = new[] { 1, 2, 3, 4, 5 }.Select(BitConverter.GetBytes).ToArray(); using (var cur = _txn.CreateCursor(_db)) cur.PutMultiple(UTF8.GetBytes("TestKey"), values); var pairs = new KeyValuePair<byte[], byte[]>[values.Length]; using (var cur = _txn.CreateCursor(_db)) { for (var i = 0; i < values.Length; i++) { cur.MoveNextDuplicate(); pairs[i] = cur.Current; } } Assert.Equal(values, pairs.Select(x => x.Value).ToArray()); }