public LightningTransaction(LightningEnvironment environment, LightningTransaction parent, TransactionBeginFlags flags) { if (environment == null) throw new ArgumentNullException("environment"); this.Environment = environment; this.ParentTransaction = parent; this.IsReadOnly = flags == TransactionBeginFlags.ReadOnly; var parentHandle = parent != null ? parent._handle : IntPtr.Zero; IntPtr handle = default(IntPtr); Native.Execute(() => Native.mdb_txn_begin(environment._handle, parentHandle, flags, out handle)); _handle = handle; this.State = LightningTransacrionState.Active; if (parent == null) this.Environment.Closing += EnvironmentOrParentTransactionClosing; else parent.Closing += EnvironmentOrParentTransactionClosing; }
public LmdbBlockTxesStorage(string baseDirectory, long blockTxesSize, int? index = null) { this.jetDirectory = Path.Combine(baseDirectory, "BlockTxes"); if (index.HasValue) this.jetDirectory = Path.Combine(jetDirectory, index.Value.ToString()); LmdbStorageManager.PrepareSparseDatabase(this.jetDirectory); this.jetInstance = new LightningEnvironment(this.jetDirectory, EnvironmentOpenFlags.NoThreadLocalStorage | EnvironmentOpenFlags.NoSync) { MaxDatabases = 10, MapSize = blockTxesSize }; this.jetInstance.Open(); using (var txn = this.jetInstance.BeginTransaction()) { globalsTableId = txn.OpenDatabase("Globals", new DatabaseOptions { Flags = DatabaseOpenFlags.Create }); blocksTableId = txn.OpenDatabase("Blocks", new DatabaseOptions { Flags = DatabaseOpenFlags.Create }); if (!txn.ContainsKey(globalsTableId, blockCountKey)) txn.Put(globalsTableId, blockCountKey, Bits.GetBytes(0)); txn.Commit(); } }
/// <summary> /// Initializes a new instance of the <see cref="LightningStorageEngine"/> class. /// </summary> /// <param name="dataPath">The database path.</param> public LightningStorageEngine(string dataPath) { DataPath = dataPath; if (!Directory.Exists(DataPath)) Directory.CreateDirectory(DataPath); _log.Info($"Data Path: {DataPath}"); _log.Info($"Compression Option: {LightningStorageUtils._compressionOption}"); var config = new EnvironmentConfiguration { MaxDatabases = MAX_DATABASES, MapSize = MAX_MAP_SIZE, MaxReaders = MAX_READERS }; _environment = new LightningEnvironment(DataPath, config); var openFlags = EnvironmentOpenFlags.WriteMap | EnvironmentOpenFlags.NoMetaSync | EnvironmentOpenFlags.MapAsync | EnvironmentOpenFlags.NoThreadLocalStorage; _environment.Open(openFlags); _openDatabases = new ConcurrentDictionary<string, LightningDatabase>(); _writeOperationsQueue = new BlockingCollection<WriteOperation>(); _cancellationTokenSource = new CancellationTokenSource(); _cancellationToken = _cancellationTokenSource.Token; Task.Factory.StartNew(() => BackgroundWriter(), TaskCreationOptions.LongRunning); }
/// <summary> /// Created new instance of LightningTransaction /// </summary> /// <param name="environment">Environment.</param> /// <param name="parent">Parent transaction or null.</param> /// <param name="flags">Transaction open options.</param> public LightningTransaction(LightningEnvironment environment, LightningTransaction parent, TransactionBeginFlags flags) { if (environment == null) { throw new ArgumentNullException("environment"); } this.Environment = environment; this.ParentTransaction = parent; this.IsReadOnly = flags == TransactionBeginFlags.ReadOnly; var parentHandle = parent != null ? parent._handle : IntPtr.Zero; IntPtr handle = default(IntPtr); NativeMethods.Execute(lib => lib.mdb_txn_begin(environment._handle, parentHandle, flags, out handle)); _handle = handle; this.State = LightningTransactionState.Active; if (parent == null) { this.Environment.Closing += EnvironmentOrParentTransactionClosing; } else { parent.Closing += EnvironmentOrParentTransactionClosing; } }
public LmdbChainStateManager(string baseDirectory, long chainStateSize) { this.baseDirectory = baseDirectory; this.jetDirectory = Path.Combine(baseDirectory, "ChainState"); this.jetDatabase = Path.Combine(this.jetDirectory, "ChainState.edb"); LmdbStorageManager.PrepareSparseDatabase(this.jetDirectory); this.jetInstance = new LightningEnvironment(this.jetDirectory, EnvironmentOpenFlags.NoThreadLocalStorage | EnvironmentOpenFlags.NoSync) { MaxDatabases = 10, MapSize = chainStateSize }; this.jetInstance.Open(); using (var txn = this.jetInstance.BeginTransaction()) { globalsTableId = txn.OpenDatabase("Globals", new DatabaseOptions { Flags = DatabaseOpenFlags.Create }); headersTableId = txn.OpenDatabase("Headers", new DatabaseOptions { Flags = DatabaseOpenFlags.Create }); unspentTxTableId = txn.OpenDatabase("UnspentTx", new DatabaseOptions { Flags = DatabaseOpenFlags.Create }); blockSpentTxesTableId = txn.OpenDatabase("BlockSpentTxes", new DatabaseOptions { Flags = DatabaseOpenFlags.Create }); blockUnmintedTxesTableId = txn.OpenDatabase("BlockUnmintedTxes", new DatabaseOptions { Flags = DatabaseOpenFlags.Create }); txn.Commit(); } this.cursorCache = new DisposableCache<IChainStateCursor>(1024, createFunc: () => new LmdbChainStateCursor(this.jetDatabase, this.jetInstance, globalsTableId, headersTableId, unspentTxTableId, blockSpentTxesTableId, blockUnmintedTxesTableId), prepareAction: cursor => { // rollback any open transaction before returning the cursor to the cache if (cursor.InTransaction) cursor.RollbackTransaction(); }); }
public LmdbChainStateCursor(string jetDatabase, LightningEnvironment jetInstance, LightningDatabase globalsTableId, LightningDatabase headersTableId, LightningDatabase unspentTxTableId, LightningDatabase blockSpentTxesTableId, LightningDatabase blockUnmintedTxesTableId) { this.jetDatabase = jetDatabase; this.jetInstance = jetInstance; this.globalsTableId = globalsTableId; this.headersTableId = headersTableId; this.unspentTxTableId = unspentTxTableId; this.blockSpentTxesTableId = blockSpentTxesTableId; this.blockUnmintedTxesTableId = blockUnmintedTxesTableId; }
/// <summary> /// Created new instance of LightningTransaction /// </summary> /// <param name="environment">Environment.</param> /// <param name="parent">Parent transaction or null.</param> /// <param name="flags">Transaction open options.</param> internal LightningTransaction(LightningEnvironment environment, IntPtr handle, LightningTransaction parent, TransactionBeginFlags flags) { if (environment == null) throw new ArgumentNullException("environment"); this.Environment = environment; this.ParentTransaction = parent; this.IsReadOnly = flags == TransactionBeginFlags.ReadOnly; this.State = LightningTransactionState.Active; _handle = handle; _subTransactionsManager = new TransactionManager(environment, this); _cursorManager = new CursorManager(this); }
public void TestCreateOpenDataBaseAndPut() { using (var env = new LightningEnvironment("pathtofolder")) { env.MaxDatabases = 2; env.Open(); using (var tx = env.BeginTransaction()) using (var db = tx.OpenDatabase("custom", new DatabaseConfiguration { Flags = DatabaseOpenFlags.Create })) { tx.Put(db, "hello", "world"); tx.Commit(); } } }
public void TestGet() { using (var env = new LightningEnvironment("pathtofolder")) { env.MaxDatabases = 2; env.Open(); using (var tx = env.BeginTransaction(TransactionBeginFlags.ReadOnly)) { var db = tx.OpenDatabase("custom"); var result = tx.Get(db, "hello"); Debug.Assert(result == "world"); } } }
public LightningDocumentStorageFactory(string rootFolder, long mapSize) { _rootFolder = rootFolder; LightningEnvironment = new LightningEnvironment(rootFolder); LightningEnvironment.MapSize = mapSize; LightningEnvironment.MaxDatabases = 2; LightningEnvironment.Open(); using (var txn = LightningEnvironment.BeginTransaction()) { using (txn.OpenDatabase(null, new DatabaseConfiguration { Flags = DatabaseOpenFlags.Create })) { txn.Commit(); } } }
/// <summary> /// Created new instance of LightningTransaction /// </summary> /// <param name="environment">Environment.</param> /// <param name="parent">Parent transaction or null.</param> /// <param name="flags">Transaction open options.</param> internal LightningTransaction(LightningEnvironment environment, LightningTransaction parent, TransactionBeginFlags flags) { Environment = environment ?? throw new ArgumentNullException(nameof(environment)); ParentTransaction = parent; IsReadOnly = flags == TransactionBeginFlags.ReadOnly; State = LightningTransactionState.Active; Environment.Disposing += Dispose; if (parent != null) { parent.Disposing += Dispose; parent.StateChanging += OnParentStateChanging; } var parentHandle = parent?.Handle() ?? IntPtr.Zero; mdb_txn_begin(environment.Handle(), parentHandle, flags, out _handle).ThrowOnError(); _originalHandle = _handle; }
private LightningEnvironment NewEnvironment(out LightningDatabase db, bool delete = true) { if (delete && Directory.Exists(_path)) Directory.Delete(_path, true); if (!Directory.Exists(_path)) Directory.CreateDirectory(_path); var env = new LightningEnvironment(_path, EnvironmentOpenFlags.None) { MapSize = 1024 * 1024 * 1024 * (long)10 }; env.Open(); var tx = env.BeginTransaction(); db = tx.OpenDatabase(); tx.Commit(); return env; }
/// <summary> /// Created new instance of LightningTransaction /// </summary> /// <param name="environment">Environment.</param> /// <param name="parent">Parent transaction or null.</param> /// <param name="flags">Transaction open options.</param> internal LightningTransaction(LightningEnvironment environment, LightningTransaction parent, TransactionBeginFlags flags) { if (environment == null) throw new ArgumentNullException(nameof(environment)); Environment = environment; ParentTransaction = parent; IsReadOnly = (flags & TransactionBeginFlags.ReadOnly) == TransactionBeginFlags.ReadOnly; State = LightningTransactionState.Active; Environment.Disposing += Dispose; if (parent != null) { parent.Disposing += Dispose; parent.StateChanging += OnParentStateChanging; } var parentHandle = parent?.Handle() ?? IntPtr.Zero; mdb_txn_begin(environment.Handle(), parentHandle, flags, out _handle); _originalHandle = _handle; }
public LmdbBlockStorage(string baseDirectory, long blocksSize) { this.jetDirectory = Path.Combine(baseDirectory, "Blocks"); LmdbStorageManager.PrepareSparseDatabase(this.jetDirectory); this.jetInstance = new LightningEnvironment(this.jetDirectory, EnvironmentOpenFlags.NoThreadLocalStorage | EnvironmentOpenFlags.NoSync) { MaxDatabases = 10, MapSize = blocksSize, }; this.jetInstance.Open(); using (var txn = this.jetInstance.BeginTransaction()) { globalsTableId = txn.OpenDatabase("Globals", new DatabaseOptions { Flags = DatabaseOpenFlags.Create }); blockHeadersTableId = txn.OpenDatabase("BlockHeaders", new DatabaseOptions { Flags = DatabaseOpenFlags.Create }); invalidBlocksTableId = txn.OpenDatabase("InvalidBlocks", new DatabaseOptions { Flags = DatabaseOpenFlags.Create }); txn.Commit(); } }
private static long ReadInternal(IEnumerable<uint> ids, PerfTracker perfTracker, LightningEnvironment env, LightningDatabase db) { using (var tx = env.BeginTransaction(LightningDB.TransactionBeginFlags.ReadOnly)) using (var cursor = new LightningCursor(db, tx)) { long v = 0; foreach (var id in ids) { var value = cursor.MoveTo(Encoding.UTF8.GetBytes(id.ToString("0000000000000000"))); v += value.Value.Length; //Debug.Assert(value != null); } return v; } }
private List<PerformanceRecord> WriteInternal( string operation, IEnumerator<TestData> enumerator, long itemsPerTransaction, long numberOfTransactions, PerfTracker perfTracker, rndseq Rflag, LightningEnvironment env, LightningDatabase db) { byte[] valueToWrite = null; var records = new List<PerformanceRecord>(); var sw = new Stopwatch(); LightningDB.PutOptions putflags = LightningDB.PutOptions.None; if (Rflag == rndseq.SEQ) putflags = LightningDB.PutOptions.AppendData; for (var transactions = 0; transactions < numberOfTransactions; transactions++) { sw.Restart(); using (var tx = env.BeginTransaction()) { for (var i = 0; i < itemsPerTransaction; i++) { enumerator.MoveNext(); valueToWrite = GetValueToWrite(valueToWrite, enumerator.Current.ValueSize); tx.Put(db, Encoding.UTF8.GetBytes(enumerator.Current.Id.ToString("0000000000000000")), valueToWrite, putflags); } tx.Commit(); } sw.Stop(); perfTracker.Record(sw.ElapsedMilliseconds); records.Add(new PerformanceRecord { Operation = operation, Time = DateTime.Now, Duration = sw.ElapsedMilliseconds, ProcessedItems = itemsPerTransaction }); } sw.Stop(); return records; }
internal static void PrepareSparseDatabase(string jetDirectory) { // detect windows OS var isWindows = Environment.OSVersion.Platform != PlatformID.MacOSX && Environment.OSVersion.Platform != PlatformID.Unix; if (!isWindows) return; // ensure db is created using (var jetInstance = new LightningEnvironment(jetDirectory)) jetInstance.Open(); // check if db is on NTFS filesystem var dbPath = Path.Combine(jetDirectory, "data.mdb"); var dbFileRoot = Path.GetPathRoot(dbPath); var dbDrive = DriveInfo.GetDrives().FirstOrDefault(x => x.RootDirectory.FullName == dbFileRoot); if (dbDrive != null && dbDrive.DriveFormat == "NTFS") { //TODO better way to set the sparse flag? // ensure db is sparse Process.Start(new ProcessStartInfo { FileName = "fsutil.exe", WorkingDirectory = jetDirectory, Arguments = "sparse setflag data.mdb", CreateNoWindow = true, WindowStyle = ProcessWindowStyle.Hidden }).WaitForExit(); } }