Esempio n. 1
0
        public RocksDBKeyValueStore(string path)
        {
            var options = new DbOptions()
                          .SetCreateIfMissing();

            _keyValueDb = RocksDBUtils.OpenRocksDb(options, path);
        }
Esempio n. 2
0
        /// <inheritdoc/>
        public override bool DeleteBlock(HashDigest <SHA256> blockHash)
        {
            byte[] key = BlockKey(blockHash);

            if (!(_blockIndexDb.Get(key) is byte[] blockDbNameByte))
            {
                return(false);
            }

            _rwBlockLock.EnterWriteLock();
            try
            {
                string blockDbName = RocksDBStoreBitConverter.GetString(blockDbNameByte);
                if (!_blockDbCache.TryGetValue(blockDbName, out RocksDb blockDb))
                {
                    blockDb = RocksDBUtils.OpenRocksDb(_options, BlockDbPath(blockDbName));
                    _blockDbCache.AddOrUpdate(blockDbName, blockDb);
                }

                _blockCache.Remove(blockHash);
                _blockIndexDb.Remove(key);
                blockDb.Remove(key);
                return(true);
            }
            finally
            {
                _rwBlockLock.ExitWriteLock();
            }
        }
Esempio n. 3
0
        /// <inheritdoc/>
        public override BlockDigest?GetBlockDigest(HashDigest <SHA256> blockHash)
        {
            if (_blockCache.TryGetValue(blockHash, out BlockDigest cachedDigest))
            {
                return(cachedDigest);
            }

            byte[] key = BlockKey(blockHash);
            if (!(_blockIndexDb.Get(key) is byte[] blockDbNameBytes))
            {
                return(null);
            }

            _rwBlockLock.EnterReadLock();
            try
            {
                string blockDbName = RocksDBStoreBitConverter.GetString(blockDbNameBytes);
                if (!_blockDbCache.TryGetValue(blockDbName, out RocksDb blockDb))
                {
                    blockDb = RocksDBUtils.OpenRocksDb(_options, BlockDbPath(blockDbName));
                    _blockDbCache.AddOrUpdate(blockDbName, blockDb);
                }

                byte[] blockBytes = blockDb.Get(key);

                BlockDigest blockDigest = BlockDigest.Deserialize(blockBytes);

                _blockCache.AddOrUpdate(blockHash, blockDigest);
                return(blockDigest);
            }
            finally
            {
                _rwBlockLock.ExitReadLock();
            }
        }
Esempio n. 4
0
        /// <inheritdoc/>
        public override bool DeleteTransaction(TxId txid)
        {
            byte[] key = TxKey(txid);

            if (!(_txIndexDb.Get(key) is byte[] txDbNameBytes))
            {
                return(false);
            }

            _rwTxLock.EnterWriteLock();
            try
            {
                string txDbName = RocksDBStoreBitConverter.GetString(txDbNameBytes);
                if (!_txDbCache.TryGetValue(txDbName, out RocksDb txDb))
                {
                    txDb = RocksDBUtils.OpenRocksDb(_options, TxDbPath(txDbName));
                    _txDbCache.AddOrUpdate(txDbName, txDb);
                }

                _txCache.Remove(txid);
                _txIndexDb.Remove(key);
                txDb.Remove(key);

                return(true);
            }
            finally
            {
                _rwTxLock.ExitWriteLock();
            }
        }
Esempio n. 5
0
        /// <inheritdoc/>
        public override void PutTransaction <T>(Transaction <T> tx)
        {
            if (_txCache.ContainsKey(tx.Id))
            {
                return;
            }

            byte[] key = TxKey(tx.Id);
            if (!(_txIndexDb.Get(key) is null))
            {
                return;
            }

            long   timestamp = tx.Timestamp.ToUnixTimeSeconds();
            string txDbName  = $"epoch{(int)timestamp / _txEpochUnitSeconds}";

            _rwTxLock.EnterWriteLock();
            try
            {
                if (!_txDbCache.TryGetValue(txDbName, out RocksDb txDb))
                {
                    txDb = RocksDBUtils.OpenRocksDb(_options, TxDbPath(txDbName));
                    _txDbCache.AddOrUpdate(txDbName, txDb);
                }

                txDb.Put(key, tx.Serialize(true));
                _txIndexDb.Put(key, RocksDBStoreBitConverter.GetBytes(txDbName));
                _txCache.AddOrUpdate(tx.Id, tx);
            }
            finally
            {
                _rwTxLock.ExitWriteLock();
            }
        }
Esempio n. 6
0
        /// <summary>
        /// Creates a new <seealso cref="RocksDBStore"/>.
        /// </summary>
        /// <param name="path">The path of the directory where the storage files will be saved.
        /// </param>
        /// <param name="blockCacheSize">The capacity of the block cache.</param>
        /// <param name="txCacheSize">The capacity of the transaction cache.</param>
        /// <param name="statesCacheSize">The capacity of the states cache.</param>
        /// <param name="maxTotalWalSize">The number to configure <c>max_total_wal_size</c> RocksDB
        /// option.</param>
        /// <param name="keepLogFileNum">The number to configure <c>keep_log_file_num</c> RocksDB
        /// option.</param>
        /// <param name="maxLogFileSize">The number to configure <c>max_log_file_size</c>
        /// RocksDB option.</param>
        public MonoRocksDBStore(
            string path,
            int blockCacheSize    = 512,
            int txCacheSize       = 1024,
            ulong?maxTotalWalSize = null,
            ulong?keepLogFileNum  = null,
            ulong?maxLogFileSize  = null
            )
        {
            _logger = Log.ForContext <RocksDBStore>();

            if (path is null)
            {
                throw new ArgumentNullException(nameof(path));
            }

            path = Path.GetFullPath(path);

            if (!Directory.Exists(path))
            {
                Directory.CreateDirectory(path);
            }

            _txCache    = new LruCache <TxId, object>(capacity: txCacheSize);
            _blockCache = new LruCache <HashDigest <SHA256>, BlockDigest>(capacity: blockCacheSize);

            _path    = path;
            _options = new DbOptions()
                       .SetCreateIfMissing();

            if (maxTotalWalSize is ulong maxTotalWalSizeValue)
            {
                _options = _options.SetMaxTotalWalSize(maxTotalWalSizeValue);
            }

            if (keepLogFileNum is ulong keepLogFileNumValue)
            {
                _options = _options.SetKeepLogFileNum(keepLogFileNumValue);
            }

            if (maxLogFileSize is ulong maxLogFileSizeValue)
            {
                _options = _options.SetMaxLogFileSize(maxLogFileSizeValue);
            }

            _blockDb           = RocksDBUtils.OpenRocksDb(_options, RocksDbPath(BlockDbName));
            _blockPerceptionDb =
                RocksDBUtils.OpenRocksDb(_options, RocksDbPath(BlockPerceptionDbName));
            _txDb       = RocksDBUtils.OpenRocksDb(_options, RocksDbPath(TxDbName));
            _stagedTxDb = RocksDBUtils.OpenRocksDb(_options, RocksDbPath(StagedTxDbName));

            // When opening a DB in a read-write mode, you need to specify all Column Families that
            // currently exist in a DB. https://github.com/facebook/rocksdb/wiki/Column-Families
            var chainDbColumnFamilies = GetColumnFamilies(_options, ChainDbName);

            _chainDb = RocksDBUtils.OpenRocksDb(
                _options, RocksDbPath(ChainDbName), chainDbColumnFamilies);
        }
Esempio n. 7
0
        /// <summary>
        /// Creates a new <seealso cref="RocksDBStore"/>.
        /// </summary>
        /// <param name="path">The path of the directory where the storage files will be saved.
        /// </param>
        /// <param name="blockCacheSize">The capacity of the block cache.</param>
        /// <param name="txCacheSize">The capacity of the transaction cache.</param>
        /// <param name="statesCacheSize">The capacity of the states cache.</param>
        public RocksDBStore(
            string path,
            int blockCacheSize  = 512,
            int txCacheSize     = 1024,
            int statesCacheSize = 10000
            )
        {
            _logger = Log.ForContext <RocksDBStore>();

            if (path is null)
            {
                throw new ArgumentNullException(nameof(path));
            }

            path = Path.GetFullPath(path);

            if (!Directory.Exists(path))
            {
                Directory.CreateDirectory(path);
            }

            _txCache     = new LruCache <TxId, object>(capacity: txCacheSize);
            _blockCache  = new LruCache <HashDigest <SHA256>, BlockDigest>(capacity: blockCacheSize);
            _statesCache = new LruCache <HashDigest <SHA256>, IImmutableDictionary <string, IValue> >(
                capacity: statesCacheSize
                );
            _lastStateRefCaches =
                new Dictionary <Guid, LruCache <string, Tuple <HashDigest <SHA256>, long> > >();

            _path    = path;
            _options = new DbOptions()
                       .SetCreateIfMissing();

            _blockDb    = RocksDBUtils.OpenRocksDb(_options, RocksDbPath(BlockDbName));
            _txDb       = RocksDBUtils.OpenRocksDb(_options, RocksDbPath(TxDbName));
            _stateDb    = RocksDBUtils.OpenRocksDb(_options, RocksDbPath(StateDbName));
            _stagedTxDb = RocksDBUtils.OpenRocksDb(_options, RocksDbPath(StagedTxDbName));

            // When opening a DB in a read-write mode, you need to specify all Column Families that
            // currently exist in a DB. https://github.com/facebook/rocksdb/wiki/Column-Families
            var chainDbColumnFamilies = GetColumnFamilies(_options, ChainDbName);

            _chainDb = RocksDBUtils.OpenRocksDb(
                _options, RocksDbPath(ChainDbName), chainDbColumnFamilies);

            var stateRefDbColumnFamilies = GetColumnFamilies(_options, StateRefDbName);

            _stateRefDb = RocksDBUtils.OpenRocksDb(
                _options, RocksDbPath(StateRefDbName), stateRefDbColumnFamilies);
        }
Esempio n. 8
0
        /// <inheritdoc/>
        public override void PutBlock <T>(Block <T> block)
        {
            if (_blockCache.ContainsKey(block.Hash))
            {
                return;
            }

            byte[] key = BlockKey(block.Hash);

            if (!(_blockIndexDb.Get(key) is null))
            {
                return;
            }

            long timestamp = block.Timestamp.ToUnixTimeSeconds();

            foreach (Transaction <T> tx in block.Transactions)
            {
                PutTransaction(tx);
            }

            _rwBlockLock.EnterWriteLock();
            try
            {
                string  blockDbName = $"epoch{timestamp / _blockEpochUnitSeconds}";
                RocksDb blockDb;
                lock (_blockDbCache)
                {
                    if (!_blockDbCache.TryGetValue(blockDbName, out blockDb))
                    {
                        blockDb = RocksDBUtils.OpenRocksDb(_options, BlockDbPath(blockDbName));
                        _blockDbCache.AddOrUpdate(blockDbName, blockDb);
                    }
                }

                BlockDigest digest = BlockDigest.FromBlock(block);
                byte[]      value  = digest.Serialize();
                blockDb.Put(key, value);
                _blockIndexDb.Put(key, RocksDBStoreBitConverter.GetBytes(blockDbName));
                _blockCache.AddOrUpdate(block.Hash, digest);
            }
            catch (Exception e)
            {
                LogUnexpectedException(nameof(PutBlock), e);
            }
            finally
            {
                _rwBlockLock.ExitWriteLock();
            }
        }
Esempio n. 9
0
        /// <inheritdoc/>
        public override Transaction <T> GetTransaction <T>(TxId txid)
        {
            if (_txCache.TryGetValue(txid, out object cachedTx))
            {
                return((Transaction <T>)cachedTx);
            }

            byte[] key = TxKey(txid);
            if (!(_txIndexDb.Get(key) is byte[] txDbNameBytes))
            {
                return(null);
            }

            string txDbName = RocksDBStoreBitConverter.GetString(txDbNameBytes);

            _rwTxLock.EnterReadLock();
            try
            {
                RocksDb txDb;
                lock (_txDbCache)
                {
                    if (!_txDbCache.TryGetValue(txDbName, out txDb))
                    {
                        txDb = RocksDBUtils.OpenRocksDb(_options, TxDbPath(txDbName));
                        _txDbCache.AddOrUpdate(txDbName, txDb);
                    }
                }

                byte[] txBytes = txDb.Get(key);

                Transaction <T> tx = Transaction <T> .Deserialize(txBytes, false);

                _txCache.AddOrUpdate(txid, tx);
                return(tx);
            }
            catch (Exception e)
            {
                LogUnexpectedException(nameof(GetTransaction), e);
                return(null);
            }
            finally
            {
                _rwTxLock.ExitReadLock();
            }
        }
Esempio n. 10
0
        /// <inheritdoc cref="BaseStore.DeleteBlock(BlockHash)"/>
        public override bool DeleteBlock(BlockHash blockHash)
        {
            byte[] key = BlockKey(blockHash);

            if (!(_blockIndexDb.Get(key) is byte[] blockDbNameByte))
            {
                return(false);
            }

            _rwBlockLock.EnterWriteLock();
            try
            {
                string  blockDbName = RocksDBStoreBitConverter.GetString(blockDbNameByte);
                RocksDb blockDb;
                lock (_blockDbCache)
                {
                    if (!_blockDbCache.TryGetValue(blockDbName, out blockDb))
                    {
                        blockDb = RocksDBUtils.OpenRocksDb(_options, BlockDbPath(blockDbName));
                        _blockDbCache.AddOrUpdate(blockDbName, blockDb);
                    }
                }

                _blockCache.Remove(blockHash);
                _blockIndexDb.Remove(key);
                blockDb.Remove(key);
                return(true);
            }
            catch (Exception e)
            {
                LogUnexpectedException(nameof(DeleteBlock), e);
            }
            finally
            {
                _rwBlockLock.ExitWriteLock();
            }

            return(false);
        }
Esempio n. 11
0
        /// <summary>
        /// Creates a new <seealso cref="RocksDBStore"/>.
        /// </summary>
        /// <param name="path">The path of the directory where the storage files will be saved.
        /// </param>
        /// <param name="blockCacheSize">The capacity of the block cache.</param>
        /// <param name="txCacheSize">The capacity of the transaction cache.</param>
        /// <param name="maxTotalWalSize">The number to configure <c>max_total_wal_size</c> RocksDB
        /// option.</param>
        /// <param name="keepLogFileNum">The number to configure <c>keep_log_file_num</c> RocksDB
        /// option.</param>
        /// <param name="maxLogFileSize">The number to configure <c>max_log_file_size</c>
        /// RocksDB option.</param>
        /// <param name="txEpochUnitSeconds">The interval between epochs of DB partions containing
        /// transactions.  86,400 seconds by default.</param>
        /// <param name="blockEpochUnitSeconds">The interval between epochs of DB partions
        /// containing blocks.  86,400 seconds by default.</param>
        /// <param name="dbConnectionCacheSize">The capacity of the block and transaction
        /// RocksDB connection cache. 100 by default.</param>
        public RocksDBStore(
            string path,
            int blockCacheSize        = 512,
            int txCacheSize           = 1024,
            ulong?maxTotalWalSize     = null,
            ulong?keepLogFileNum      = null,
            ulong?maxLogFileSize      = null,
            int txEpochUnitSeconds    = 86400,
            int blockEpochUnitSeconds = 86400,
            int dbConnectionCacheSize = 100
            )
        {
            _logger = Log.ForContext <RocksDBStore>();

            if (path is null)
            {
                throw new ArgumentNullException(nameof(path));
            }

            path = Path.GetFullPath(path);

            if (!Directory.Exists(path))
            {
                Directory.CreateDirectory(path);
            }

            _txCache    = new LruCache <TxId, object>(capacity: txCacheSize);
            _blockCache = new LruCache <BlockHash, BlockDigest>(capacity: blockCacheSize);

            _path = path;
            _txEpochUnitSeconds = txEpochUnitSeconds > 0
                ? txEpochUnitSeconds
                : throw new ArgumentException(
                                            "It must be greater than 0.",
                                            nameof(txEpochUnitSeconds));
            _blockEpochUnitSeconds = blockEpochUnitSeconds > 0
                ? blockEpochUnitSeconds
                : throw new ArgumentException(
                                               "It must be greater than 0.",
                                               nameof(blockEpochUnitSeconds));
            _options = new DbOptions()
                       .SetCreateIfMissing();

            if (maxTotalWalSize is ulong maxTotalWalSizeValue)
            {
                _options = _options.SetMaxTotalWalSize(maxTotalWalSizeValue);
            }

            if (keepLogFileNum is ulong keepLogFileNumValue)
            {
                _options = _options.SetKeepLogFileNum(keepLogFileNumValue);
            }

            if (maxLogFileSize is ulong maxLogFileSizeValue)
            {
                _options = _options.SetMaxLogFileSize(maxLogFileSizeValue);
            }

            _blockIndexDb      = RocksDBUtils.OpenRocksDb(_options, BlockDbPath(BlockIndexDbName));
            _blockPerceptionDb =
                RocksDBUtils.OpenRocksDb(_options, RocksDbPath(BlockPerceptionDbName));
            _txIndexDb     = RocksDBUtils.OpenRocksDb(_options, TxDbPath(TxIndexDbName));
            _stagedTxDb    = RocksDBUtils.OpenRocksDb(_options, RocksDbPath(StagedTxDbName));
            _txExecutionDb =
                RocksDBUtils.OpenRocksDb(_options, RocksDbPath(TxExecutionDbName));
            _txIdBlockHashIndexDb =
                RocksDBUtils.OpenRocksDb(_options, RocksDbPath(TxIdBlockHashIndexDbName));

            // When opening a DB in a read-write mode, you need to specify all Column Families that
            // currently exist in a DB. https://github.com/facebook/rocksdb/wiki/Column-Families
            var chainDbColumnFamilies = GetColumnFamilies(_options, ChainDbName);

            _chainDb = RocksDBUtils.OpenRocksDb(
                _options, RocksDbPath(ChainDbName), chainDbColumnFamilies);

            _rwTxLock    = new ReaderWriterLockSlim(LockRecursionPolicy.SupportsRecursion);
            _rwBlockLock = new ReaderWriterLockSlim(LockRecursionPolicy.SupportsRecursion);

            _blockDbCache = new LruCache <string, RocksDb>(dbConnectionCacheSize);
            _blockDbCache.SetPreRemoveDataMethod(db =>
            {
                db.Dispose();
                return(true);
            });
            _txDbCache = new LruCache <string, RocksDb>(dbConnectionCacheSize);
            _txDbCache.SetPreRemoveDataMethod(db =>
            {
                db.Dispose();
                return(true);
            });
        }