public LightningTransaction(LightningEnvironment environment, LightningTransaction parent, TransactionBeginFlags flags)
{
if (environment == null)
throw new ArgumentNullException("environment");
this.Environment = environment;
this.ParentTransaction = parent;
this.IsReadOnly = flags == TransactionBeginFlags.ReadOnly;
var parentHandle = parent != null
? parent._handle
: IntPtr.Zero;
IntPtr handle = default(IntPtr);
Native.Execute(() => Native.mdb_txn_begin(environment._handle, parentHandle, flags, out handle));
_handle = handle;
this.State = LightningTransacrionState.Active;
if (parent == null)
this.Environment.Closing += EnvironmentOrParentTransactionClosing;
else
parent.Closing += EnvironmentOrParentTransactionClosing;
}
public LmdbBlockTxesStorage(string baseDirectory, long blockTxesSize, int? index = null)
{
this.jetDirectory = Path.Combine(baseDirectory, "BlockTxes");
if (index.HasValue)
this.jetDirectory = Path.Combine(jetDirectory, index.Value.ToString());
LmdbStorageManager.PrepareSparseDatabase(this.jetDirectory);
this.jetInstance = new LightningEnvironment(this.jetDirectory, EnvironmentOpenFlags.NoThreadLocalStorage | EnvironmentOpenFlags.NoSync)
{
MaxDatabases = 10,
MapSize = blockTxesSize
};
this.jetInstance.Open();
using (var txn = this.jetInstance.BeginTransaction())
{
globalsTableId = txn.OpenDatabase("Globals", new DatabaseOptions { Flags = DatabaseOpenFlags.Create });
blocksTableId = txn.OpenDatabase("Blocks", new DatabaseOptions { Flags = DatabaseOpenFlags.Create });
if (!txn.ContainsKey(globalsTableId, blockCountKey))
txn.Put(globalsTableId, blockCountKey, Bits.GetBytes(0));
txn.Commit();
}
}
/// <summary>
/// Initializes a new instance of the <see cref="LightningStorageEngine"/> class.
/// </summary>
/// <param name="dataPath">The database path.</param>
public LightningStorageEngine(string dataPath)
{
DataPath = dataPath;
if (!Directory.Exists(DataPath))
Directory.CreateDirectory(DataPath);
_log.Info($"Data Path: {DataPath}");
_log.Info($"Compression Option: {LightningStorageUtils._compressionOption}");
var config = new EnvironmentConfiguration
{
MaxDatabases = MAX_DATABASES,
MapSize = MAX_MAP_SIZE,
MaxReaders = MAX_READERS
};
_environment = new LightningEnvironment(DataPath, config);
var openFlags = EnvironmentOpenFlags.WriteMap |
EnvironmentOpenFlags.NoMetaSync |
EnvironmentOpenFlags.MapAsync |
EnvironmentOpenFlags.NoThreadLocalStorage;
_environment.Open(openFlags);
_openDatabases = new ConcurrentDictionary<string, LightningDatabase>();
_writeOperationsQueue = new BlockingCollection<WriteOperation>();
_cancellationTokenSource = new CancellationTokenSource();
_cancellationToken = _cancellationTokenSource.Token;
Task.Factory.StartNew(() => BackgroundWriter(), TaskCreationOptions.LongRunning);
}
/// <summary>
/// Created new instance of LightningTransaction
/// </summary>
/// <param name="environment">Environment.</param>
/// <param name="parent">Parent transaction or null.</param>
/// <param name="flags">Transaction open options.</param>
public LightningTransaction(LightningEnvironment environment, LightningTransaction parent, TransactionBeginFlags flags)
{
if (environment == null)
{
throw new ArgumentNullException("environment");
}
this.Environment = environment;
this.ParentTransaction = parent;
this.IsReadOnly = flags == TransactionBeginFlags.ReadOnly;
var parentHandle = parent != null
? parent._handle
: IntPtr.Zero;
IntPtr handle = default(IntPtr);
NativeMethods.Execute(lib => lib.mdb_txn_begin(environment._handle, parentHandle, flags, out handle));
_handle = handle;
this.State = LightningTransactionState.Active;
if (parent == null)
{
this.Environment.Closing += EnvironmentOrParentTransactionClosing;
}
else
{
parent.Closing += EnvironmentOrParentTransactionClosing;
}
}
public LmdbChainStateManager(string baseDirectory, long chainStateSize)
{
this.baseDirectory = baseDirectory;
this.jetDirectory = Path.Combine(baseDirectory, "ChainState");
this.jetDatabase = Path.Combine(this.jetDirectory, "ChainState.edb");
LmdbStorageManager.PrepareSparseDatabase(this.jetDirectory);
this.jetInstance = new LightningEnvironment(this.jetDirectory, EnvironmentOpenFlags.NoThreadLocalStorage | EnvironmentOpenFlags.NoSync)
{
MaxDatabases = 10,
MapSize = chainStateSize
};
this.jetInstance.Open();
using (var txn = this.jetInstance.BeginTransaction())
{
globalsTableId = txn.OpenDatabase("Globals", new DatabaseOptions { Flags = DatabaseOpenFlags.Create });
headersTableId = txn.OpenDatabase("Headers", new DatabaseOptions { Flags = DatabaseOpenFlags.Create });
unspentTxTableId = txn.OpenDatabase("UnspentTx", new DatabaseOptions { Flags = DatabaseOpenFlags.Create });
blockSpentTxesTableId = txn.OpenDatabase("BlockSpentTxes", new DatabaseOptions { Flags = DatabaseOpenFlags.Create });
blockUnmintedTxesTableId = txn.OpenDatabase("BlockUnmintedTxes", new DatabaseOptions { Flags = DatabaseOpenFlags.Create });
txn.Commit();
}
this.cursorCache = new DisposableCache<IChainStateCursor>(1024,
createFunc: () => new LmdbChainStateCursor(this.jetDatabase, this.jetInstance, globalsTableId, headersTableId, unspentTxTableId, blockSpentTxesTableId, blockUnmintedTxesTableId),
prepareAction: cursor =>
{
// rollback any open transaction before returning the cursor to the cache
if (cursor.InTransaction)
cursor.RollbackTransaction();
});
}
public LmdbChainStateCursor(string jetDatabase, LightningEnvironment jetInstance, LightningDatabase globalsTableId, LightningDatabase headersTableId, LightningDatabase unspentTxTableId, LightningDatabase blockSpentTxesTableId, LightningDatabase blockUnmintedTxesTableId)
{
this.jetDatabase = jetDatabase;
this.jetInstance = jetInstance;
this.globalsTableId = globalsTableId;
this.headersTableId = headersTableId;
this.unspentTxTableId = unspentTxTableId;
this.blockSpentTxesTableId = blockSpentTxesTableId;
this.blockUnmintedTxesTableId = blockUnmintedTxesTableId;
}
/// <summary>
/// Created new instance of LightningTransaction
/// </summary>
/// <param name="environment">Environment.</param>
/// <param name="parent">Parent transaction or null.</param>
/// <param name="flags">Transaction open options.</param>
internal LightningTransaction(LightningEnvironment environment, IntPtr handle, LightningTransaction parent, TransactionBeginFlags flags)
{
if (environment == null)
throw new ArgumentNullException("environment");
this.Environment = environment;
this.ParentTransaction = parent;
this.IsReadOnly = flags == TransactionBeginFlags.ReadOnly;
this.State = LightningTransactionState.Active;
_handle = handle;
_subTransactionsManager = new TransactionManager(environment, this);
_cursorManager = new CursorManager(this);
}
public void TestCreateOpenDataBaseAndPut()
{
using (var env = new LightningEnvironment("pathtofolder"))
{
env.MaxDatabases = 2;
env.Open();
using (var tx = env.BeginTransaction())
using (var db = tx.OpenDatabase("custom", new DatabaseConfiguration { Flags = DatabaseOpenFlags.Create }))
{
tx.Put(db, "hello", "world");
tx.Commit();
}
}
}
public void TestGet()
{
using (var env = new LightningEnvironment("pathtofolder"))
{
env.MaxDatabases = 2;
env.Open();
using (var tx = env.BeginTransaction(TransactionBeginFlags.ReadOnly))
{
var db = tx.OpenDatabase("custom");
var result = tx.Get(db, "hello");
Debug.Assert(result == "world");
}
}
}
public LightningDocumentStorageFactory(string rootFolder, long mapSize)
{
_rootFolder = rootFolder;
LightningEnvironment = new LightningEnvironment(rootFolder);
LightningEnvironment.MapSize = mapSize;
LightningEnvironment.MaxDatabases = 2;
LightningEnvironment.Open();
using (var txn = LightningEnvironment.BeginTransaction())
{
using (txn.OpenDatabase(null, new DatabaseConfiguration { Flags = DatabaseOpenFlags.Create }))
{
txn.Commit();
}
}
}
/// <summary>
/// Created new instance of LightningTransaction
/// </summary>
/// <param name="environment">Environment.</param>
/// <param name="parent">Parent transaction or null.</param>
/// <param name="flags">Transaction open options.</param>
internal LightningTransaction(LightningEnvironment environment, LightningTransaction parent, TransactionBeginFlags flags)
{
Environment = environment ?? throw new ArgumentNullException(nameof(environment));
ParentTransaction = parent;
IsReadOnly = flags == TransactionBeginFlags.ReadOnly;
State = LightningTransactionState.Active;
Environment.Disposing += Dispose;
if (parent != null)
{
parent.Disposing += Dispose;
parent.StateChanging += OnParentStateChanging;
}
var parentHandle = parent?.Handle() ?? IntPtr.Zero;
mdb_txn_begin(environment.Handle(), parentHandle, flags, out _handle).ThrowOnError();
_originalHandle = _handle;
}
private LightningEnvironment NewEnvironment(out LightningDatabase db, bool delete = true)
{
if (delete && Directory.Exists(_path))
Directory.Delete(_path, true);
if (!Directory.Exists(_path))
Directory.CreateDirectory(_path);
var env = new LightningEnvironment(_path, EnvironmentOpenFlags.None)
{
MapSize = 1024 * 1024 * 1024 * (long)10
};
env.Open();
var tx = env.BeginTransaction();
db = tx.OpenDatabase();
tx.Commit();
return env;
}
/// <summary>
/// Created new instance of LightningTransaction
/// </summary>
/// <param name="environment">Environment.</param>
/// <param name="parent">Parent transaction or null.</param>
/// <param name="flags">Transaction open options.</param>
internal LightningTransaction(LightningEnvironment environment, LightningTransaction parent, TransactionBeginFlags flags)
{
if (environment == null)
throw new ArgumentNullException(nameof(environment));
Environment = environment;
ParentTransaction = parent;
IsReadOnly = (flags & TransactionBeginFlags.ReadOnly) == TransactionBeginFlags.ReadOnly;
State = LightningTransactionState.Active;
Environment.Disposing += Dispose;
if (parent != null)
{
parent.Disposing += Dispose;
parent.StateChanging += OnParentStateChanging;
}
var parentHandle = parent?.Handle() ?? IntPtr.Zero;
mdb_txn_begin(environment.Handle(), parentHandle, flags, out _handle);
_originalHandle = _handle;
}
public LmdbBlockStorage(string baseDirectory, long blocksSize)
{
this.jetDirectory = Path.Combine(baseDirectory, "Blocks");
LmdbStorageManager.PrepareSparseDatabase(this.jetDirectory);
this.jetInstance = new LightningEnvironment(this.jetDirectory, EnvironmentOpenFlags.NoThreadLocalStorage | EnvironmentOpenFlags.NoSync)
{
MaxDatabases = 10,
MapSize = blocksSize,
};
this.jetInstance.Open();
using (var txn = this.jetInstance.BeginTransaction())
{
globalsTableId = txn.OpenDatabase("Globals", new DatabaseOptions { Flags = DatabaseOpenFlags.Create });
blockHeadersTableId = txn.OpenDatabase("BlockHeaders", new DatabaseOptions { Flags = DatabaseOpenFlags.Create });
invalidBlocksTableId = txn.OpenDatabase("InvalidBlocks", new DatabaseOptions { Flags = DatabaseOpenFlags.Create });
txn.Commit();
}
}
private static long ReadInternal(IEnumerable<uint> ids, PerfTracker perfTracker, LightningEnvironment env,
LightningDatabase db)
{
using (var tx = env.BeginTransaction(LightningDB.TransactionBeginFlags.ReadOnly))
using (var cursor = new LightningCursor(db, tx))
{
long v = 0;
foreach (var id in ids)
{
var value = cursor.MoveTo(Encoding.UTF8.GetBytes(id.ToString("0000000000000000")));
v += value.Value.Length;
//Debug.Assert(value != null);
}
return v;
}
}
private List<PerformanceRecord> WriteInternal(
string operation,
IEnumerator<TestData> enumerator,
long itemsPerTransaction,
long numberOfTransactions,
PerfTracker perfTracker,
rndseq Rflag,
LightningEnvironment env,
LightningDatabase db)
{
byte[] valueToWrite = null;
var records = new List<PerformanceRecord>();
var sw = new Stopwatch();
LightningDB.PutOptions putflags = LightningDB.PutOptions.None;
if (Rflag == rndseq.SEQ)
putflags = LightningDB.PutOptions.AppendData;
for (var transactions = 0; transactions < numberOfTransactions; transactions++)
{
sw.Restart();
using (var tx = env.BeginTransaction())
{
for (var i = 0; i < itemsPerTransaction; i++)
{
enumerator.MoveNext();
valueToWrite = GetValueToWrite(valueToWrite, enumerator.Current.ValueSize);
tx.Put(db, Encoding.UTF8.GetBytes(enumerator.Current.Id.ToString("0000000000000000")), valueToWrite, putflags);
}
tx.Commit();
}
sw.Stop();
perfTracker.Record(sw.ElapsedMilliseconds);
records.Add(new PerformanceRecord
{
Operation = operation,
Time = DateTime.Now,
Duration = sw.ElapsedMilliseconds,
ProcessedItems = itemsPerTransaction
});
}
sw.Stop();
return records;
}
internal static void PrepareSparseDatabase(string jetDirectory)
{
// detect windows OS
var isWindows = Environment.OSVersion.Platform != PlatformID.MacOSX && Environment.OSVersion.Platform != PlatformID.Unix;
if (!isWindows)
return;
// ensure db is created
using (var jetInstance = new LightningEnvironment(jetDirectory))
jetInstance.Open();
// check if db is on NTFS filesystem
var dbPath = Path.Combine(jetDirectory, "data.mdb");
var dbFileRoot = Path.GetPathRoot(dbPath);
var dbDrive = DriveInfo.GetDrives().FirstOrDefault(x => x.RootDirectory.FullName == dbFileRoot);
if (dbDrive != null && dbDrive.DriveFormat == "NTFS")
{
//TODO better way to set the sparse flag?
// ensure db is sparse
Process.Start(new ProcessStartInfo
{
FileName = "fsutil.exe",
WorkingDirectory = jetDirectory,
Arguments = "sparse setflag data.mdb",
CreateNoWindow = true,
WindowStyle = ProcessWindowStyle.Hidden
}).WaitForExit();
}
}