public MultiProcessTests(SharedFileSystem fileSystem)
{
var path = fileSystem.CreateNewDirectoryForTest();
_env = new LightningEnvironment(path);
_env.Open();
}
public void DatabaseDeleteShouldRemoveAllDuplicateDataItems()
{
var fs = new SharedFileSystem();
using (var env = new LightningEnvironment(fs.CreateNewDirectoryForTest(), configuration: new EnvironmentConfiguration {
MapSize = 1024 * 1024, MaxDatabases = 1
}))
{
env.Open();
using (var tx = env.BeginTransaction())
using (var db = tx.OpenDatabase(configuration: new DatabaseConfiguration()
{
Flags = DatabaseOpenFlags.DuplicatesSort
}))
{
var key = "key";
var value1 = "value1";
var value2 = "value2";
tx.Put(db, key, value1);
tx.Put(db, key, value2);
tx.Delete(db, key);
Assert.False(tx.ContainsKey(db, key));
}
}
}
public TransactionTests(SharedFileSystem fileSystem)
{
var path = fileSystem.CreateNewDirectoryForTest();
_env = new LightningEnvironment(path);
_env.Open();
}
public static T WithAutogrowth <T>(this LightningEnvironment env, Func <T> action)
{
try
{
return(action());
}
catch (LightningException ex) when(ex.StatusCode == LightningDB.Native.Lmdb.MDB_MAP_FULL)
{
if (Monitor.TryEnter(env))
{
try
{
env.MapSize = (env.MapSize == 0 ? 1048576L : env.MapSize) << 1;
return(env.WithAutogrowthRetry(action));
}
finally
{
Monitor.Exit(env);
}
}
else
{
lock (env)
{
return(env.WithAutogrowthRetry(action));
}
}
}
}
public (TimeSpan openTime, long memorySize) Open()
{
Enviroment.Dispose();
_enviroment = null;
GC.Collect(); GC.WaitForPendingFinalizers(); GC.Collect();
var memStart = GC.GetTotalMemory(true);
var stopwatch = Stopwatch.StartNew();
Enviroment.Open();
using (var tx = Enviroment.BeginTransaction())
using (tx.OpenDatabase(configuration: new DatabaseConfiguration {
Flags = DatabaseOpenFlags.None
}))
{
stopwatch.Stop();
GC.Collect(); GC.WaitForPendingFinalizers(); GC.Collect();
var memFinish = GC.GetTotalMemory(false);
var memDif = (memFinish - memStart);
return(openTime : stopwatch.Elapsed, memorySize : memDif / 1024);
}
}
public void GlobalSetup()
{
Console.WriteLine("Global Setup Begin");
const string Path = "TestDirectory";
if (Directory.Exists(Path))
{
Directory.Delete(Path, true);
}
Env = new LightningEnvironment(Path)
{
MaxDatabases = 1
};
Env.Open();
using (var tx = Env.BeginTransaction()) {
DB = tx.OpenDatabase();
tx.Commit();
}
RunSetup();
Console.WriteLine("Global Setup End");
}
public void Init()
{
Directory.CreateDirectory(_path);
_env = new LightningEnvironment(_path, EnvironmentOpenFlags.None);
_env.Open();
}
public LightningPersistence(LightningConfig config)
{
_config = config.Clone();
_cts = new CancellationTokenSource();
Console.WriteLine($"ThreadId ctor: {Thread.CurrentThread.ManagedThreadId}");
Env = new LightningEnvironment(_config.Name ?? "db")
{
MaxDatabases = _config.MaxTables > 0u ? (int)_config.MaxTables : 20,
MapSize = (_config.StorageLimit > 0L ? (long)_config.StorageLimit : 10L) * 1024 * 1024 * 1024
};
EnvironmentOpenFlags envFlags;
switch (config.SyncMode)
{
case LightningDbSyncMode.Fsync:
envFlags = EnvironmentOpenFlags.NoThreadLocalStorage;
break;
case LightningDbSyncMode.Async:
envFlags = EnvironmentOpenFlags.NoThreadLocalStorage | EnvironmentOpenFlags.WriteMap | EnvironmentOpenFlags.MapAsync;
break;
case LightningDbSyncMode.NoSync:
envFlags = EnvironmentOpenFlags.NoThreadLocalStorage | EnvironmentOpenFlags.NoSync;
break;
case LightningDbSyncMode.ReadOnly:
envFlags = EnvironmentOpenFlags.NoThreadLocalStorage | EnvironmentOpenFlags.ReadOnly;
break;
default:
throw new ArgumentOutOfRangeException();
}
Env.Open(envFlags);
_envHandle = GCHandle.Alloc(Env);
_tables = new ConcurrentDictionary <string, Table>();
_dupTables = new ConcurrentDictionary <string, DupTable>();
_writeQueue = new BlockingCollection <Delegates>();
var threadStart = new ThreadStart(() =>
{
foreach (var batch in DequeueBatch(_writeQueue))
{
ProcessBatch(batch);
}
_writeTaskCompletion.SetResult(null);
});
var writeThread = new Thread(threadStart)
{
Name = "LMDB Writer thread",
IsBackground = true
};
writeThread.Start();
}
private LightningEnvironment GetLightningEnvironment()
{
var t0 = Stopwatch.StartNew();
var env = new LightningEnvironment(_dir)
{
MapSize = _mapSize,
MaxDatabases = 1
};
env.Open(EnvironmentOpenFlags.NoLock);
using (var tx = env.BeginTransaction())
using (tx.OpenDatabase(DatabaseName, new DatabaseConfiguration {
Flags = DatabaseOpenFlags.Create
}))
{
tx.Commit();
}
t0.Stop();
if (_log.IsDebugEnabled)
{
_log.Debug($"Init of LMDB in directory [{_dir}] took [{t0.ElapsedMilliseconds} ms]");
}
return(env);
}
public void CreateEnvironmentWithAutoResize()
{
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
using (var env = new LightningEnvironment(_path, new EnvironmentConfiguration
{
MapSize = 1048576,
AutoResizeWindows = true,
}))
{
env.Open();
}
using (var env = new LightningEnvironment(_path, new EnvironmentConfiguration
{
MapSize = 1048576,
AutoResizeWindows = true,
}))
{
env.Open();
}
using (var dbFile = File.OpenRead(Path.Combine(_path, "data.mdb")))
{
Assert.Equal(8192, dbFile.Length);
}
}
}
public void Dispose()
{
if (_env != null && _env.IsOpened)
_env.Dispose();
_env = null;
}
private LightningEnvironment NewEnvironment(out LightningDatabase db, bool delete = true)
{
if (delete && Directory.Exists(_path))
{
Directory.Delete(_path, true);
}
if (!Directory.Exists(_path))
{
Directory.CreateDirectory(_path);
}
var env = new LightningEnvironment(_path, EnvironmentOpenFlags.None)
{
MapSize = 1024 * 1024 * 1024 * (long)10
};
env.Open();
var tx = env.BeginTransaction();
db = tx.OpenDatabase();
tx.Commit();
return(env);
}
public void Init()
{
Directory.CreateDirectory(_path);
_env = new LightningEnvironment(_path, EnvironmentOpenFlags.None);
_env.Open();
}
public static void DbWrite()
{
using (var env = new LightningEnvironment("db"))
{
env.MaxDatabases = 20;
env.MapSize = 20L * 1024 * 1024 * 1024;
env.Open(EnvironmentOpenFlags.NoThreadLocalStorage /*| EnvironmentOpenFlags.NoLock */);
using (var db = CreateLightningDatabase(env, "custom9"))
{
Task runner = Task.Factory.StartNew(() => RunParallelRead(db, "outer", 10, 30), TaskCreationOptions.LongRunning);
using (var tx = env.BeginTransaction())
{
Console.WriteLine("Write opened");
for (int i = 0; i < 1000; i++)
{
tx.Put(db, Encoding.UTF8.GetBytes($"hello1 + {i}"), Encoding.UTF8.GetBytes("world"));
}
tx.Commit();
Console.WriteLine("Write finished");
Thread.Sleep(1000);
ReadBlock(db, "main");
}
runner.Wait();
}
}
}
public void Dispose()
{
if (_env != null)
_env.Dispose();
_env = null;
}
public void TestIterator()
{
using (var env = new LightningEnvironment(DbEnv))
{
env.MaxDatabases = 2;
env.Open();
using (var tx = env.BeginTransaction())
using (var _db = tx.OpenDatabase(DbName, new DatabaseConfiguration {
Flags = DatabaseOpenFlags.Create
}))
//assert
using (var cur = tx.CreateCursor(_db))
{
while (cur.MoveNext())
{
var key = ByteHelper.GetStringFromBytesASCI(cur.Current.Key);
var value = ByteHelper.GetStringFromBytesASCI(cur.Current.Value);
Console.WriteLine(key);
}
}
}
}
public void EnvironmentShouldBeOpened()
{
_env = new LightningEnvironment(_path);
_env.Open();
Assert.True(_env.IsOpened);
}
public void EnvironmentShouldBeOpened()
{
_env = new LightningEnvironment(_path);
_env.Open();
Assert.True(_env.IsOpened);
}
public void Dispose()
{
if (_env != null)
{
_env.Dispose();
_env = null;
}
}
public void EnvironmentShouldBeCreatedIfReadOnly()
{
_env = new LightningEnvironment(_path);
_env.Open(); //readonly requires environment to have been created at least once before
_env.Dispose();
_env = new LightningEnvironment(_path);
_env.Open(EnvironmentOpenFlags.ReadOnly);
}
public static void RunTransactionScenario(this LightningEnvironment env,
Action <LightningTransaction, LightningDatabase> scenario,
DatabaseOpenFlags flags = DatabaseOpenFlags.Create, TransactionBeginFlags transactionFlags = TransactionBeginFlags.None)
{
using var tx = env.BeginTransaction(transactionFlags);
using var db = tx.OpenDatabase(configuration: new DatabaseConfiguration { Flags = flags });
scenario(tx, db);
}
public LightningStore(string pathToFolder, string dbName)
{
_env = new LightningEnvironment(pathToFolder);
_dbName = dbName;
_env.MaxDatabases = 1;
_env.Open();
_pool = new SpanPool <byte>(PoolSizeInBytes);
}
public FinalizeTests(SharedFileSystem fileSystem)
{
var env = new LightningEnvironment(fileSystem.CreateNewDirectoryForTest());
env.Open();
var tx = env.BeginTransaction();
var db = tx.OpenDatabase();
tx.CreateCursor(db);
}
public LightningEnvironment GetDatabase(string name)
{
var folder = Machine.GetServiceFolder(Id.Service, name);
var db = new LightningEnvironment(folder);
_simulationResources.Push(db);
return(db);
}
public void EnvironmentShouldBeCreatedIfReadOnly()
{
_env = new LightningEnvironment(_path);
_env.Open(); //readonly requires environment to have been created at least once before
_env.Dispose();
_env = new LightningEnvironment(_path);
_env.Open(EnvironmentOpenFlags.ReadOnly);
}
public LightningDbPersistence(LightningDbSettings settings)
{
_environment = settings.ToEnvironment();
_environment.Open(EnvironmentOpenFlags.WriteMap | EnvironmentOpenFlags.NoSync);
OpenDatabase(Outgoing);
OpenDatabase(Delayed);
}
static LightningEnvironment BuildEnvironment(string path)
{
var env = new LightningEnvironment(path);
env.MaxDatabases = 2;
env.Open();
return(env);
}
public Lightning(string envPath, int numOfDb)
{
env = new LightningEnvironment(envPath)
{
MaxDatabases = numOfDb
};
env.Open();
}
public void EnvironmentShouldBeClosed()
{
_env = new LightningEnvironment(_path);
_env.Open();
_env.Dispose();
Assert.False(_env.IsOpened);
}
public void Dispose()
{
if (_env != null && _env.IsOpened)
{
_env.Dispose();
}
_env = null;
}
public void CanOpenEnvironmentMoreThan50Mb()
{
_env = new LightningEnvironment(_path)
{
MapSize = 55 * 1024 * 1024
};
_env.Open();
}
public void EnvironmentShouldBeClosed()
{
_env = new LightningEnvironment(_path);
_env.Open();
_env.Dispose();
Assert.False(_env.IsOpened);
}
public void EnvironmentShouldBeCreatedIfReadOnly()
{
//arrange
//act
_env = new LightningEnvironment(_path, EnvironmentOpenFlags.ReadOnly);
//assert
}
public void EnvironmentShouldBeCreatedIfWithoutFlags()
{
//arrange
//act
_env = new LightningEnvironment(_path, EnvironmentOpenFlags.None);
//assert
}
public ChangeStream(string dirPath)
{
_env = new LightningEnvironment(dirPath);
_env.Open();
using (var tx = _env.BeginTransaction())
{
_db = tx.OpenDatabase(configuration: s_defaultDbConfig);
}
}
public void EnvironmentShouldBeCreatedIfWithoutFlags()
{
//arrange
//act
_env = new LightningEnvironment(_path, EnvironmentOpenFlags.None);
//assert
}
public void EnvironmentShouldBeCreatedIfReadOnly()
{
//arrange
//act
_env = new LightningEnvironment(_path, EnvironmentOpenFlags.ReadOnly);
//assert
}
public FinalizeTests(SharedFileSystem fileSystem)
{
var env = new LightningEnvironment(fileSystem.CreateNewDirectoryForTest());
env.Open();
var tx = env.BeginTransaction();
var db = tx.OpenDatabase();
tx.CreateCursor(db);
}
public void EnvironmentShouldBeCopied(bool compact)
{
_env = new LightningEnvironment(_path);
_env.Open();
_env.CopyTo(_pathCopy, compact);
if (Directory.GetFiles(_pathCopy).Length == 0)
Assert.True(false, "Copied files doesn't exist");
}
public void Init()
{
Directory.CreateDirectory(_path);
_env = new LightningEnvironment(_path, EnvironmentOpenFlags.None);
_env.Open();
_txn = _env.BeginTransaction();
_db = _txn.OpenDatabase();
}
public void Init()
{
Directory.CreateDirectory(_path);
_env = new LightningEnvironment(_path, EnvironmentOpenFlags.None);
_env.Open();
_txn = _env.BeginTransaction();
_db = _txn.OpenDatabase();
}
public CursorTests(SharedFileSystem fileSystem)
{
var path = fileSystem.CreateNewDirectoryForTest();
_env = new LightningEnvironment(path);
_env.Open();
_txn = _env.BeginTransaction();
}
public DatabaseIOTests(SharedFileSystem fileSystem)
{
var path = fileSystem.CreateNewDirectoryForTest();
_env = new LightningEnvironment(path);
_env.MaxDatabases = 2;
_env.Open();
_txn = _env.BeginTransaction();
_db = _txn.OpenDatabase(configuration: new DatabaseConfiguration {Flags = DatabaseOpenFlags.Create});
}
public void EnvironmentCreatedFromConfig()
{
var mapExpected = 1024*1024*20;
var maxDatabaseExpected = 2;
var maxReadersExpected = 3;
var config = new EnvironmentConfiguration {MapSize = mapExpected, MaxDatabases = maxDatabaseExpected, MaxReaders = maxReadersExpected};
_env = new LightningEnvironment(_path, config);
Assert.Equal(_env.MapSize, mapExpected);
Assert.Equal(_env.MaxDatabases, maxDatabaseExpected);
Assert.Equal(_env.MaxReaders, maxReadersExpected);
}
public void EnvironmentShouldBeOpened()
{
//arrange
_env = new LightningEnvironment(_path, EnvironmentOpenFlags.None);
//act
_env.Open();
//assert
Assert.AreEqual(true, _env.IsOpened);
}
internal void Configure(LightningEnvironment env)
{
if (_mapSize.HasValue)
env.MapSize = _mapSize.Value;
if (_maxDatabases.HasValue)
env.MaxDatabases = _maxDatabases.Value;
if (_maxReaders.HasValue)
env.MaxReaders = _maxReaders.Value;
}
public void RemotePut(string environment)
{
var env = new LightningEnvironment(environment);
env.Open();
using (env)
using(var tx = env.BeginTransaction())
using(var db = tx.OpenDatabase())
{
tx.Put(db, Encoding.UTF8.GetBytes("hello"), Encoding.UTF8.GetBytes("world"));
tx.Commit();
}
}
public void MaxDatabasesWorksThroughConfigIssue62()
{
var config = new EnvironmentConfiguration { MaxDatabases = 2 };
_env = new LightningEnvironment(_path, config);
_env.Open();
using (var tx = _env.BeginTransaction())
{
tx.OpenDatabase("db1", new DatabaseConfiguration {Flags = DatabaseOpenFlags.Create});
tx.OpenDatabase("db2", new DatabaseConfiguration {Flags = DatabaseOpenFlags.Create});
tx.Commit();
}
Assert.Equal(_env.MaxDatabases, 2);
}
public void Cleanup()
{
if (_env != null && _env.IsOpened)
_env.Close();
_env = null;
if (Directory.Exists(_path))
Directory.Delete(_path, true);
if (Directory.Exists(_pathCopy))
Directory.Delete(_pathCopy, true);
}
public TransactionManager(LightningEnvironment environment, LightningTransaction parentTransaction)
{
if (environment == null)
throw new ArgumentNullException("environment");
_environment = environment;
_parentTransaction = parentTransaction;
_parentHandle = parentTransaction != null
? parentTransaction._handle
: IntPtr.Zero;
_transactions = new ConcurrentDictionary<LightningTransaction, bool>();
_comparatorsStore = new ConcurrentDictionary<CompareFunction, bool>();
}
private static void DoTest(bool useRandomKeys = false)
{
var env = new LightningEnvironment(CreateNewDirectoryForTest());
env.MaxDatabases = 2;
env.MapSize = 1000 * 1024 * 1025; // 100 MB
env.Open();
try
{
DoTestImpl(env, useRandomKeys);
}
finally
{
if (env != null)
env.Dispose();
}
}
public MultiProcessTests(SharedFileSystem fileSystem)
{
var path = fileSystem.CreateNewDirectoryForTest();
_env = new LightningEnvironment(path);
_env.Open();
}
public void CanOpenEnvironmentMoreThan50Mb()
{
_env = new LightningEnvironment(_path)
{
MapSize = 55 * 1024 * 1024
};
_env.Open();
}
public void EnvironmentShouldBeCreatedIfWithoutFlags()
{
_env = new LightningEnvironment(_path);
_env.Open();
}
public void StartingTransactionBeforeEnvironmentOpen()
{
_env = new LightningEnvironment(_path);
Assert.Throws<InvalidOperationException>(() => _env.BeginTransaction());
}
public void CanLoadAndDisposeMultipleEnvironments()
{
_env = new LightningEnvironment(_path);
_env.Dispose();
_env = new LightningEnvironment(_path);
}
public DatabaseTests(SharedFileSystem fileSystem)
{
var path = fileSystem.CreateNewDirectoryForTest();
_env = new LightningEnvironment(path);
}
public void CanOpenEnvironmentMoreThan50Mb()
{
//arrange
_env = new LightningEnvironment(_path, EnvironmentOpenFlags.None)
{
MapSize = 55 * 1024 * 1024
};
//act-assert
_env.Open();
}
public void EnvironmentShouldBeCopied()
{
//arrange
_env = new LightningEnvironment(_path, EnvironmentOpenFlags.None);
_env.Open();
//act
_env.CopyTo(_pathCopy);
//assert
if (Directory.GetFiles(_pathCopy).Length == 0)
Assert.Fail("Copied files doesn't exist");
}
private static void DoTestImpl(LightningEnvironment env, bool useRandomKeys = false)
{
var numItemsToWrite = 1 * 1000; // One thousand
//var numItemsToWrite = 10 * 1000; // Ten thousand
//var numItemsToWrite = 100 * 1000; // One hundred thousand
//var numItemsToWrite = 1 * 1000 * 1000; // 1 million
//var numItemsToWrite = 10 * 1000 * 1000; // 10 million
var randon = new Random(1773);
Console.WriteLine("Using {0} keys", useRandomKeys ? "RANDOM" : "SEQUENTIAL");
var writeTimer = Stopwatch.StartNew();
// Need to specify DatabaseOpenFlags.IntegerKey if we want the items to be sorted, not entirely sure why though,
// probably big/little endian related, see http://www.openldap.org/lists/openldap-bugs/201308/msg00050.html
var dbConfig = new DatabaseConfiguration { Flags = DatabaseOpenFlags.Create | DatabaseOpenFlags.IntegerKey };
using (var tx = env.BeginTransaction())
using (var db = tx.OpenDatabase("test", dbConfig))
{
for (var i = 0; i < numItemsToWrite; ++i)
{
var key = i;
if (useRandomKeys)
key = randon.Next();
//tx.Put(db, BitConverter.GetBytes(i), BitConverter.GetBytes(i));
var text = "Some Text plus: 'Key' = " + key.ToString("N0"); // + " " + new String('c', 1000);
var data = GetBinaryEncodedString(text);
tx.Put(db, BitConverter.GetBytes(key), data);
}
tx.Commit();
var stats = tx.GetStats(db);
Console.WriteLine("Stats\n # Entries: {0:N0}\n Depth: {1}\n Page Size: {2}",
stats.ms_entries.ToInt64(), stats.ms_depth, stats.ms_psize);
Console.WriteLine(" Branch Pages: {0:N0}\n Leaf Pages: {1}\n Overflow Pages: {2}",
stats.ms_branch_pages.ToInt64(), stats.ms_leaf_pages.ToInt64(), stats.ms_overflow_pages.ToInt64());
}
writeTimer.Stop();
Console.WriteLine("Took {0,10:N2} ms ({1}) to WRITE {2,9:N0} values ({3,10:N0} WRITES/sec)",
writeTimer.Elapsed.TotalMilliseconds, writeTimer.Elapsed, numItemsToWrite,
numItemsToWrite / writeTimer.Elapsed.TotalMilliseconds * 1000.0);
var readTimer = Stopwatch.StartNew();
var readCounter = 0;
using (var tx = env.BeginTransaction())
using (var db = tx.OpenDatabase("test"))
{
tx.Get(db, BitConverter.GetBytes(int.MinValue));
ValueStructure currentKey = default(ValueStructure);
ValueStructure currentValue = default(ValueStructure);
using (var cursor = tx.CreateCursor(db))
{
while (cursor.MoveNext())
{
//var current = cursor.Current;
cursor.GetCurrent(out currentKey, out currentValue);
unsafe
{
var keyData = *((int*)(currentKey.data.ToPointer()));
int valueLengthSize;
var ptr = (byte*)currentValue.data.ToPointer();
var length = Read7BitEncodedInt(ptr, out valueLengthSize);
var text = new string((sbyte*)(ptr + valueLengthSize), 0, length, Encoding.UTF8);
//Console.WriteLine("{{ Key: {0:N0}, Value: \"{1}\" }}", keyData, text);
}
readCounter++;
}
}
}
readTimer.Stop();
Console.WriteLine("Took {0,10:N2} ms ({1}) to READ {2,9:N0} values ({3,10:N0} READS/sec)",
readTimer.Elapsed.TotalMilliseconds, readTimer.Elapsed, readCounter,
readCounter / readTimer.Elapsed.TotalMilliseconds * 1000.0);
Console.WriteLine();
}
