public void RoundTrip()
{
var span = MemoryMarshal.AsBytes(MemoryMarshal.CreateSpan(ref RoundTripKey, 1));
var key = span.ToArray();
using (var tx = _env.BeginTransaction())
{
using (var db = tx.OpenDatabase(_dbName,
new DatabaseConfiguration {
Flags = DatabaseOpenFlags.Create
}))
{
tx.Put(db, key, key);
tx.Commit();
}
}
using (var tx = _env.BeginTransaction())
{
using (var db = tx.OpenDatabase(_dbName,
new DatabaseConfiguration {
Flags = DatabaseOpenFlags.Create
}))
{
tx.Delete(db, key);
tx.Commit();
}
}
}
public bool WriteEvents(List <EventTransaction> eventTransactions)
{
try
{
using (var tx = _environment.BeginTransaction())
using (var eventDb = tx.OpenDatabase(EventDb, new DatabaseConfiguration {
Flags = DatabaseOpenFlags.Create | DatabaseOpenFlags.IntegerKey
}))
using (var aggregateIndex = tx.OpenDatabase(AggregateIndex, new DatabaseConfiguration {
Flags = DatabaseOpenFlags.Create | DatabaseOpenFlags.DuplicatesSort
}))
{
foreach (var transaction in eventTransactions)
{
foreach (var @event in transaction.Events)
{
byte[] nextSerialNumber = BitConverter.GetBytes(@event.SerialId);
byte[] aggregateId = @event.AggregateId.ToByteArray();
var serializedEvent = _serializer.Serialize(@event);
tx.Put(eventDb, nextSerialNumber, serializedEvent, PutOptions.AppendData);
tx.Put(aggregateIndex, aggregateId, nextSerialNumber, PutOptions.AppendDuplicateData);
}
}
tx.Commit();
}
}catch (Exception)
{
return(false);
}
return(true);
}
private void DoStuffHelper()
{
using (var tx = _env.BeginTransaction())
using (var db = tx.OpenDatabase("test", new DatabaseConfiguration {
Flags = DatabaseOpenFlags.Create
}))
{
for (var i = 0; i < 100; ++i)
{
tx.Put(db, BitConverter.GetBytes(i), BitConverter.GetBytes(i));
}
tx.Commit();
}
using (var tx = _env.BeginTransaction())
using (var db = tx.OpenDatabase("test"))
{
tx.Get(db, BitConverter.GetBytes(1));
using (var cursor = tx.CreateCursor(db))
{
while (cursor.MoveNext())
{
var current = cursor.Current;
}
}
}
}
public void Enqueue(byte[][] values)
{
if (values == null)
{
throw new ArgumentNullException(nameof(values));
}
lock (_sync)
{
RequireNotDisposed();
var totalPayloadWritten = 0UL;
using (var tx = _env.BeginTransaction())
using (var db = tx.OpenDatabase())
{
foreach (var v in values)
{
if (v == null)
{
throw new ArgumentException("Value array may not contain null.");
}
tx.Put(db, ULongKeyToByteKey(_nextId++), v);
totalPayloadWritten += (ulong)v.Length;
}
tx.Commit();
_entries += (ulong)values.Length;
_writtenSinceRotateCheck += totalPayloadWritten;
}
RotateIfRequired();
}
}
public void StoreIncomingMessages(params Message[] messages)
{
using (var tx = _environment.BeginTransaction())
{
StoreIncomingMessages(tx, messages);
tx.Commit();
}
}
public void TransactionShouldBeCreated()
{
//arrange
//act
_txn = _env.BeginTransaction();
//assert
Assert.AreEqual(LightningTransactionState.Active, _txn.State);
}
public byte[] Get(byte[] key)
{
using (var tx = env.BeginTransaction(TransactionBeginFlags.ReadOnly))
using (var db = tx.OpenDatabase(DbName))
{
var result = tx.Get(db, key);
tx.Commit();
return(result);
}
}
public Task <long> GetLengthAsync(CancellationToken cancellationToken = default)
{
if (cancellationToken.IsCancellationRequested)
{
return(Task.FromResult(0L));
}
using var tx = Env.BeginTransaction(TransactionBeginFlags.ReadOnly);
using var db = tx.OpenDatabase(configuration: Config);
var count = tx.GetEntriesCount(db); // entries also contains handles to databases
return(Task.FromResult(count));
}
public long GetCounter()
{
using (var tx = _le.BeginTransaction(TransactionBeginFlags.ReadOnly)) {
var key = FdbTuple.Create((byte)Tables.SysCounter);
var val = tx.Get(_ld, key.GetBytes());
if (val == null)
{
return(0);
}
return(BitConverter.ToInt64(val, 0));
}
}
public void DatabaseShouldBeCreated()
{
var dbName = "test";
_env.MaxDatabases = 2;
_env.Open();
_txn = _env.BeginTransaction();
//arrange
//act
_txn.OpenDatabase(dbName, DatabaseOpenFlags.Create);
//assert
}
public KeyValuePair <long, byte[]> GetLastCheckpoint()
{
using (var tx = _env.BeginTransaction(TransactionBeginFlags.ReadOnly))
using (var c = tx.CreateCursor(_db))
{
if (c.MoveToLast())
{
return(Convert(c.Current));
}
else
{
return(new KeyValuePair <long, byte[]>(-1L, null));
}
}
}
private LightningEnvironment GetLightningEnvironment()
{
var t0 = Stopwatch.StartNew();
var env = new LightningEnvironment(_dir)
{
MapSize = _mapSize,
MaxDatabases = 1
};
env.Open(EnvironmentOpenFlags.NoLock);
using (var tx = env.BeginTransaction())
using (tx.OpenDatabase(DatabaseName, new DatabaseConfiguration {
Flags = DatabaseOpenFlags.Create
}))
{
tx.Commit();
}
t0.Stop();
if (_log.IsDebugEnabled)
{
_log.Debug($"Init of LMDB in directory [{_dir}] took [{t0.ElapsedMilliseconds} ms]");
}
return(env);
}
public void ResetData()
{
using (var tx = _env.BeginTransaction()) {
_db.Truncate(tx);
tx.Commit();
}
}
public void DatabaseDeleteShouldRemoveAllDuplicateDataItems()
{
var fs = new SharedFileSystem();
using (var env = new LightningEnvironment(fs.CreateNewDirectoryForTest(), configuration: new EnvironmentConfiguration {
MapSize = 1024 * 1024, MaxDatabases = 1
}))
{
env.Open();
using (var tx = env.BeginTransaction())
using (var db = tx.OpenDatabase(configuration: new DatabaseConfiguration()
{
Flags = DatabaseOpenFlags.DuplicatesSort
}))
{
var key = "key";
var value1 = "value1";
var value2 = "value2";
tx.Put(db, key, value1);
tx.Put(db, key, value2);
tx.Delete(db, key);
Assert.False(tx.ContainsKey(db, key));
}
}
}
public void TestIterator()
{
using (var env = new LightningEnvironment(DbEnv))
{
env.MaxDatabases = 2;
env.Open();
using (var tx = env.BeginTransaction())
using (var _db = tx.OpenDatabase(DbName, new DatabaseConfiguration {
Flags = DatabaseOpenFlags.Create
}))
//assert
using (var cur = tx.CreateCursor(_db))
{
while (cur.MoveNext())
{
var key = ByteHelper.GetStringFromBytesASCI(cur.Current.Key);
var value = ByteHelper.GetStringFromBytesASCI(cur.Current.Value);
Console.WriteLine(key);
}
}
}
}
public static void DbWrite()
{
using (var env = new LightningEnvironment("db"))
{
env.MaxDatabases = 20;
env.MapSize = 20L * 1024 * 1024 * 1024;
env.Open(EnvironmentOpenFlags.NoThreadLocalStorage /*| EnvironmentOpenFlags.NoLock */);
using (var db = CreateLightningDatabase(env, "custom9"))
{
Task runner = Task.Factory.StartNew(() => RunParallelRead(db, "outer", 10, 30), TaskCreationOptions.LongRunning);
using (var tx = env.BeginTransaction())
{
Console.WriteLine("Write opened");
for (int i = 0; i < 1000; i++)
{
tx.Put(db, Encoding.UTF8.GetBytes($"hello1 + {i}"), Encoding.UTF8.GetBytes("world"));
}
tx.Commit();
Console.WriteLine("Write finished");
Thread.Sleep(1000);
ReadBlock(db, "main");
}
runner.Wait();
}
}
}
private LightningEnvironment NewEnvironment(out LightningDatabase db, bool delete = true)
{
if (delete && Directory.Exists(_path))
{
Directory.Delete(_path, true);
}
if (!Directory.Exists(_path))
{
Directory.CreateDirectory(_path);
}
var env = new LightningEnvironment(_path, EnvironmentOpenFlags.None)
{
MapSize = 1024 * 1024 * 1024 * (long)10
};
env.Open();
var tx = env.BeginTransaction();
db = tx.OpenDatabase();
tx.Commit();
return(env);
}
public void GlobalSetup()
{
Console.WriteLine("Global Setup Begin");
const string Path = "TestDirectory";
if (Directory.Exists(Path))
{
Directory.Delete(Path, true);
}
Env = new LightningEnvironment(Path)
{
MaxDatabases = 1
};
Env.Open();
using (var tx = Env.BeginTransaction()) {
DB = tx.OpenDatabase();
tx.Commit();
}
RunSetup();
Console.WriteLine("Global Setup End");
}
public FinalizeTests(SharedFileSystem fileSystem)
{
var env = new LightningEnvironment(fileSystem.CreateNewDirectoryForTest());
env.Open();
var tx = env.BeginTransaction();
var db = tx.OpenDatabase();
tx.CreateCursor(db);
}
public static void RunTransactionScenario(this LightningEnvironment env,
Action <LightningTransaction, LightningDatabase> scenario,
DatabaseOpenFlags flags = DatabaseOpenFlags.Create, TransactionBeginFlags transactionFlags = TransactionBeginFlags.None)
{
using var tx = env.BeginTransaction(transactionFlags);
using var db = tx.OpenDatabase(configuration: new DatabaseConfiguration { Flags = flags });
scenario(tx, db);
}
public Task CreateAsync(params IIdentityEntity[] documents)
{
using (var tx = _env.BeginTransaction())
{
using (var db = tx.OpenDatabase())
{
foreach (var document in documents)
{
var content = JsonConvert.SerializeObject(document.Entity, _jsonSettings);
tx.Put(db, Encoding.UTF8.GetBytes(document.Id.ToString()), Encoding.UTF8.GetBytes(content));
}
tx.Commit();
}
}
return(Task.CompletedTask);
}
public void CanDeletePreviouslyCommittedWithMultipleValuesByPassingNullForValue()
{
_env.RunTransactionScenario((tx, db) =>
{
var key = MemoryMarshal.Cast <char, byte>("abcd");
tx.Put(db, key, MemoryMarshal.Cast <char, byte>("Value1"));
tx.Put(db, key, MemoryMarshal.Cast <char, byte>("Value2"), PutOptions.AppendData);
tx.Commit();
tx.Dispose();
using var delTxn = _env.BeginTransaction();
var result = delTxn.Delete(db, key, null);
Assert.Equal(MDBResultCode.Success, result);
result = delTxn.Commit();
Assert.Equal(MDBResultCode.Success, result);
}, DatabaseOpenFlags.Create | DatabaseOpenFlags.DuplicatesFixed);
}
public ChangeStream(string dirPath)
{
_env = new LightningEnvironment(dirPath);
_env.Open();
using (var tx = _env.BeginTransaction())
{
_db = tx.OpenDatabase(configuration: s_defaultDbConfig);
}
}
public ObjectRepositoryTransaction <TKey, T> BeginTransaction(bool readOnly = false)
{
var tx = _env.BeginTransaction(readOnly ? TransactionBeginFlags.ReadOnly : TransactionBeginFlags.None);
return(new ObjectRepositoryTransaction <TKey, T>(
_settings,
tx,
_db));
}
public static LmdbEnv CreateDb(string folder, long size,
EnvironmentOpenFlags flags = EnvironmentOpenFlags.None)
{
if (!Directory.Exists(folder))
{
Directory.CreateDirectory(folder);
}
var env = new LightningEnvironment(folder, new EnvironmentConfiguration()
{
MaxDatabases = 1,
MapSize = size,
MaxReaders = 1024 * 4
});
var readOnly = (flags & EnvironmentOpenFlags.ReadOnly) == EnvironmentOpenFlags.ReadOnly;
env.Open(flags);
LightningDatabase db;
if (readOnly)
{
using (var tx = env.BeginTransaction(TransactionBeginFlags.ReadOnly)) {
db = tx.OpenDatabase();
tx.Commit();
}
}
else
{
var config = new DatabaseConfiguration()
{
Flags = DatabaseOpenFlags.Create
};
using (var tx = env.BeginTransaction()) {
db = tx.OpenDatabase(configuration: config);
tx.Commit();
}
}
return(new LmdbEnv(db, env));
}
public void Init()
{
Directory.CreateDirectory(_path);
_env = new LightningEnvironment(_path, EnvironmentOpenFlags.None);
_env.Open();
_txn = _env.BeginTransaction();
_db = _txn.OpenDatabase();
}
public void Init()
{
Directory.CreateDirectory(_path);
_env = new LightningEnvironment(_path, EnvironmentOpenFlags.None);
_env.Open();
_txn = _env.BeginTransaction();
_db = _txn.OpenDatabase();
}
public FinalizeTests(SharedFileSystem fileSystem)
{
var env = new LightningEnvironment(fileSystem.CreateNewDirectoryForTest());
env.Open();
var tx = env.BeginTransaction();
var db = tx.OpenDatabase();
tx.CreateCursor(db);
}
public CursorTests(SharedFileSystem fileSystem)
{
var path = fileSystem.CreateNewDirectoryForTest();
_env = new LightningEnvironment(path);
_env.Open();
_txn = _env.BeginTransaction();
}
public CursorTests(SharedFileSystem fileSystem)
{
var path = fileSystem.CreateNewDirectoryForTest();
_env = new LightningEnvironment(path);
_env.Open();
_txn = _env.BeginTransaction();
}
public ObjectRepository(ObjectRepositorySettings <TKey, T> settings)
{
_settings = settings;
_env = new LightningEnvironment(settings.Path);
_env.Open();
using (var tx = _env.BeginTransaction())
{
_db = tx.OpenDatabase();
}
}
private List <PerformanceRecord> WriteInternal(
string operation,
IEnumerator <TestData> enumerator,
long itemsPerTransaction,
long numberOfTransactions,
PerfTracker perfTracker,
rndseq Rflag,
LightningEnvironment env,
LightningDatabase db)
{
byte[] valueToWrite = null;
var records = new List <PerformanceRecord>();
var sw = new Stopwatch();
LightningDB.PutOptions putflags = LightningDB.PutOptions.None;
if (Rflag == rndseq.SEQ)
{
putflags = LightningDB.PutOptions.AppendData;
}
for (var transactions = 0; transactions < numberOfTransactions; transactions++)
{
sw.Restart();
using (var tx = env.BeginTransaction())
{
for (var i = 0; i < itemsPerTransaction; i++)
{
enumerator.MoveNext();
valueToWrite = GetValueToWrite(valueToWrite, enumerator.Current.ValueSize);
tx.Put(db, Encoding.UTF8.GetBytes(enumerator.Current.Id.ToString("0000000000000000")), valueToWrite, putflags);
}
tx.Commit();
}
sw.Stop();
perfTracker.Record(sw.ElapsedMilliseconds);
records.Add(new PerformanceRecord
{
Operation = operation,
Time = DateTime.Now,
Duration = sw.ElapsedMilliseconds,
ProcessedItems = itemsPerTransaction
});
}
sw.Stop();
return(records);
}
public DatabaseIOTests(SharedFileSystem fileSystem)
{
var path = fileSystem.CreateNewDirectoryForTest();
_env = new LightningEnvironment(path);
_env.MaxDatabases = 2;
_env.Open();
_txn = _env.BeginTransaction();
_db = _txn.OpenDatabase(configuration: new DatabaseConfiguration {Flags = DatabaseOpenFlags.Create});
}
public void RemotePut(string environment)
{
var env = new LightningEnvironment(environment);
env.Open();
using (env)
using(var tx = env.BeginTransaction())
using(var db = tx.OpenDatabase())
{
tx.Put(db, Encoding.UTF8.GetBytes("hello"), Encoding.UTF8.GetBytes("world"));
tx.Commit();
}
}
public void Enqueue(byte[][] values)
{
lock (_sync)
{
RequireNotDisposed();
var totalPayloadWritten = 0UL;
using (var tx = _env.BeginTransaction())
using (var db = tx.OpenDatabase())
{
foreach (var v in values)
{
tx.Put(db, ULongKeyToByteKey(_nextId++), v);
totalPayloadWritten += (ulong)v.Length;
}
tx.Commit();
_entries += (ulong)values.Length;
_writtenSinceRotateCheck += totalPayloadWritten;
}
RotateIfRequired();
}
}
public void MaxDatabasesWorksThroughConfigIssue62()
{
var config = new EnvironmentConfiguration { MaxDatabases = 2 };
_env = new LightningEnvironment(_path, config);
_env.Open();
using (var tx = _env.BeginTransaction())
{
tx.OpenDatabase("db1", new DatabaseConfiguration {Flags = DatabaseOpenFlags.Create});
tx.OpenDatabase("db2", new DatabaseConfiguration {Flags = DatabaseOpenFlags.Create});
tx.Commit();
}
Assert.Equal(_env.MaxDatabases, 2);
}
public void CanCountEnvironmentEntries()
{
const int entriesCount = 10;
//arrange
_env = new LightningEnvironment(_path, EnvironmentOpenFlags.None);
_env.Open();
using (var txn = _env.BeginTransaction())
using (var db = txn.OpenDatabase(null, new DatabaseOptions { Flags = DatabaseOpenFlags.None }))
{
for (var i = 0; i < entriesCount; i++)
txn.Put(db, i.ToString(), i.ToString());
txn.Commit();
}
//act
var count = _env.EntriesCount;
//assert;
Assert.AreEqual(entriesCount, count);
}
public void StartingTransactionBeforeEnvironmentOpen()
{
_env = new LightningEnvironment(_path);
Assert.Throws<InvalidOperationException>(() => _env.BeginTransaction());
}
public void CanGetUsedSize()
{
const int entriesCount = 1;
//arrange
_env = new LightningEnvironment(_path, EnvironmentOpenFlags.None);
_env.Open();
var initialUsedSize = _env.UsedSize;
using (var txn = _env.BeginTransaction())
using (var db = txn.OpenDatabase(null, new DatabaseOptions { Flags = DatabaseOpenFlags.None }))
{
for (int i = 0; i < entriesCount; i++)
txn.Put(db, i, i);
txn.Commit();
}
//act
var sizeDelta = _env.UsedSize - initialUsedSize;
//act-assert;
Assert.AreEqual(_env.PageSize, sizeDelta);
}
private static void DoTestImpl(LightningEnvironment env, bool useRandomKeys = false)
{
var numItemsToWrite = 1 * 1000; // One thousand
//var numItemsToWrite = 10 * 1000; // Ten thousand
//var numItemsToWrite = 100 * 1000; // One hundred thousand
//var numItemsToWrite = 1 * 1000 * 1000; // 1 million
//var numItemsToWrite = 10 * 1000 * 1000; // 10 million
var randon = new Random(1773);
Console.WriteLine("Using {0} keys", useRandomKeys ? "RANDOM" : "SEQUENTIAL");
var writeTimer = Stopwatch.StartNew();
// Need to specify DatabaseOpenFlags.IntegerKey if we want the items to be sorted, not entirely sure why though,
// probably big/little endian related, see http://www.openldap.org/lists/openldap-bugs/201308/msg00050.html
var dbConfig = new DatabaseConfiguration { Flags = DatabaseOpenFlags.Create | DatabaseOpenFlags.IntegerKey };
using (var tx = env.BeginTransaction())
using (var db = tx.OpenDatabase("test", dbConfig))
{
for (var i = 0; i < numItemsToWrite; ++i)
{
var key = i;
if (useRandomKeys)
key = randon.Next();
//tx.Put(db, BitConverter.GetBytes(i), BitConverter.GetBytes(i));
var text = "Some Text plus: 'Key' = " + key.ToString("N0"); // + " " + new String('c', 1000);
var data = GetBinaryEncodedString(text);
tx.Put(db, BitConverter.GetBytes(key), data);
}
tx.Commit();
var stats = tx.GetStats(db);
Console.WriteLine("Stats\n # Entries: {0:N0}\n Depth: {1}\n Page Size: {2}",
stats.ms_entries.ToInt64(), stats.ms_depth, stats.ms_psize);
Console.WriteLine(" Branch Pages: {0:N0}\n Leaf Pages: {1}\n Overflow Pages: {2}",
stats.ms_branch_pages.ToInt64(), stats.ms_leaf_pages.ToInt64(), stats.ms_overflow_pages.ToInt64());
}
writeTimer.Stop();
Console.WriteLine("Took {0,10:N2} ms ({1}) to WRITE {2,9:N0} values ({3,10:N0} WRITES/sec)",
writeTimer.Elapsed.TotalMilliseconds, writeTimer.Elapsed, numItemsToWrite,
numItemsToWrite / writeTimer.Elapsed.TotalMilliseconds * 1000.0);
var readTimer = Stopwatch.StartNew();
var readCounter = 0;
using (var tx = env.BeginTransaction())
using (var db = tx.OpenDatabase("test"))
{
tx.Get(db, BitConverter.GetBytes(int.MinValue));
ValueStructure currentKey = default(ValueStructure);
ValueStructure currentValue = default(ValueStructure);
using (var cursor = tx.CreateCursor(db))
{
while (cursor.MoveNext())
{
//var current = cursor.Current;
cursor.GetCurrent(out currentKey, out currentValue);
unsafe
{
var keyData = *((int*)(currentKey.data.ToPointer()));
int valueLengthSize;
var ptr = (byte*)currentValue.data.ToPointer();
var length = Read7BitEncodedInt(ptr, out valueLengthSize);
var text = new string((sbyte*)(ptr + valueLengthSize), 0, length, Encoding.UTF8);
//Console.WriteLine("{{ Key: {0:N0}, Value: \"{1}\" }}", keyData, text);
}
readCounter++;
}
}
}
readTimer.Stop();
Console.WriteLine("Took {0,10:N2} ms ({1}) to READ {2,9:N0} values ({3,10:N0} READS/sec)",
readTimer.Elapsed.TotalMilliseconds, readTimer.Elapsed, readCounter,
readCounter / readTimer.Elapsed.TotalMilliseconds * 1000.0);
Console.WriteLine();
}