public RocksDbCacheClient(RpcClient rpcClient, string cachePath)
            {
                this.rpcClient = rpcClient;

                if (!Directory.Exists(cachePath))
                {
                    Directory.CreateDirectory(cachePath);
                }
                var columnFamilies = GetColumnFamilies(cachePath);

                db = RocksDb.Open(new DbOptions().SetCreateIfMissing(true), cachePath, columnFamilies);
예제 #2
0
        public Snapshot(Store store, RocksDb db)
        {
            this.store = store;
            this.db = db;
            this.snapshot = db.CreateSnapshot();
            this.batch = new WriteBatch();

            options = new ReadOptions();
            options.SetFillCache(false);
            options.SetSnapshot(snapshot);
        }
예제 #3
0
        public DbOnTheRocks(string dbPath, IDbConfig dbConfig, ILogManager logManager = null) // TODO: check column families
        {
            ILogger logger = logManager?.GetClassLogger();

            if (!Directory.Exists(dbPath))
            {
                Directory.CreateDirectory(dbPath);
            }

            if (logger != null)
            {
                if (logger.IsInfo)
                {
                    logger.Info($"Using database directory {dbPath}");
                }
            }

            if (dbPath.EndsWith(StateDbPath))
            {
                _dbInstance = DbInstance.State;
            }
            else if (dbPath.EndsWith(BlockInfosDbPath))
            {
                _dbInstance = DbInstance.BlockInfos;
            }
            else if (dbPath.EndsWith(BlocksDbPath))
            {
                _dbInstance = DbInstance.Blocks;
            }
            else if (dbPath.EndsWith(CodeDbPath))
            {
                _dbInstance = DbInstance.Code;
            }
            else if (dbPath.EndsWith(ReceiptsDbPath))
            {
                _dbInstance = DbInstance.Receipts;
            }
            else if (dbPath.EndsWith(PendingTxsDbPath))
            {
                _dbInstance = DbInstance.PendingTxs;
            }
            else if (dbPath.EndsWith(TraceDbPath))
            {
                _dbInstance = DbInstance.Trace;
            }
            else
            {
                _dbInstance = DbInstance.Other;
            }

            DbOptions options = BuildOptions(dbConfig, _dbInstance);

            _db = DbsByPath.GetOrAdd(dbPath, path => RocksDb.Open(options, path));
        }
예제 #4
0
        public RocksDbContext(string path = "ChainLachain")
        {
            _dbpath       = path;
            _writeOptions = new WriteOptions();
            _writeOptions.DisableWal(0);
            _writeOptions.SetSync(true);

            var options = new DbOptions().SetCreateIfMissing();

            _rocksDb = RocksDb.Open(options, _dbpath);
        }
예제 #5
0
        public LocalStateRepository(ILogger <LocalStateRepository> logger,
                                    IOptions <RuntimeDirectory> runtimeDirOptions)
        {
            _logger           = logger;
            _runtimeDirectory = runtimeDirOptions;
            _formatter        = new BinaryFormatter();

            var options = new DbOptions().SetCreateIfMissing(true);

            _dbInstance = RocksDb.Open(options, Path.Combine(_runtimeDirectory.Value.LocalStateRepositoryDb, "LocalOptions.db"));
        }
예제 #6
0
        public RocksDbChainStore(Network network, DataFolder dataFolder, ChainIndexer chainIndexer)
        {
            this.dataFolder   = dataFolder.ChainPath;
            this.network      = network;
            this.ChainIndexer = chainIndexer;
            this.headers      = new MemoryCountCache <uint256, BlockHeader>(601);
            this.locker       = new object();

            this.dbOptions = new DbOptions().SetCreateIfMissing(true);
            this.rocksDb   = RocksDb.Open(this.dbOptions, this.dataFolder);
        }
예제 #7
0
        public RocksDBLogStore(RaftStateMachine rn, string path)
        {
            this.stateMachine = rn;
            string logdbFile = path + "_logdb.db";
            var    options   = new DbOptions()
                               .SetCreateIfMissing(true);

            db = RocksDb.Open(options, logdbFile);

            this.ReloadFromStorage();
        }
 /// <summary>
 /// Lists the column families from <see cref="RocksDbSharp.RocksDb"/> located at <paramref name="rocksdbPath"/>.
 /// </summary>
 /// <param name="rocksdbPath">The path of <see cref="RocksDbSharp.RocksDb"/> to load.</param>
 public void List([Option] string? rocksdbPath = null)
 {
     rocksdbPath ??= Directory.GetCurrentDirectory();
     var options = new DbOptions();
     var columnFamilies = RocksDb.ListColumnFamilies(
         options,
         rocksdbPath);
     foreach (var columnFamily in columnFamilies)
     {
         _inputOutputErrorContainer.Out.WriteLine(columnFamily);
     }
 }
예제 #9
0
        internal static RocksDb OpenRocksDb(
            DbOptions options, string dbPath, ColumnFamilies?columnFamilies = null)
        {
            if (!Directory.Exists(dbPath))
            {
                Directory.CreateDirectory(dbPath);
            }

            return(columnFamilies is null
                ? RocksDb.Open(options, dbPath)
                : RocksDb.Open(options, dbPath, columnFamilies));
        }
예제 #10
0
 private ColumnFamilyHandle CreateColumnFamilyIfMissing(RocksDb db, string columnFamily)
 {
     try
     {
         return(db.GetColumnFamily(columnFamily));
     }
     catch
     {
         var options = new ColumnFamilyOptions();
         return(db.CreateColumnFamily(options, columnFamily));
     }
 }
예제 #11
0
        /// <summary>
        /// Materialize a <see cref="InMemoryKeyValueStore"/> with the given name.
        /// </summary>
        /// <typeparam name="KS">New serializer for <typeparamref name="K"/> type</typeparam>
        /// <typeparam name="VS">New serializer for <typeparamref name="V"/> type</typeparam>
        /// <param name="storeName">the name of the underlying <see cref="IKTable{K, V}"/> state store; valid characters are ASCII alphanumerics, '.', '_' and '-'.</param>
        /// <returns>a new <see cref="InMemory{K, V}"/> instance with the given storeName</returns>
        public static RocksDb <K, V> @As <KS, VS>(string storeName)
            where KS : ISerDes <K>, new()
            where VS : ISerDes <V>, new()
        {
            var m = new RocksDb <K, V>(storeName, new RocksDbKeyValueBytesStoreSupplier(storeName))
            {
                KeySerdes   = new KS(),
                ValueSerdes = new VS()
            };

            return(m);
        }
예제 #12
0
        public void Initialize(string connection, string database)
        {
            if (String.IsNullOrEmpty(connection))
            {
                throw new ArgumentNullException(nameof(connection), "No connection / path provided for RocksDB");
            }

            //Connection = path in rocksDB, but I don't like this - we need to rethink how we could
            //Make this signature more generic for all of the varieties of repositories
            var options = new DbOptions().SetCreateIfMissing(true);

            _rocksDb = RocksDb.Open(options, connection);
        }
예제 #13
0
        private bool disposedValue = false; // To detect redundant calls

        protected virtual void Dispose(bool disposing)
        {
            if (!disposedValue)
            {
                if (disposing)
                {
                    _dbInstance.Dispose();
                }

                _dbInstance   = null;
                disposedValue = true;
            }
        }
예제 #14
0
        private bool disposedValue = false; // To detect redundant calls

        protected virtual void Dispose(bool disposing)
        {
            if (!disposedValue)
            {
                if (disposing)
                {
                    _preAggDatabase?.Dispose();
                }

                _preAggDatabase = null;
                disposedValue   = true;
            }
        }
예제 #15
0
        public RocksdbChainStore(Network network, DataFolder dataFolder, ChainIndexer chainIndexer)
        {
            this.network        = network;
            this.ChainIndexer   = chainIndexer;
            this.nearTipHeaders = new MemoryCountCache <uint256, BlockHeader>(601);
            this.recentHeaders  = new MemoryCountCache <uint256, BlockHeader>(100);
            this.locker         = new object();

            // Open a connection to a new DB and create if not found
            var options = new DbOptions().SetCreateIfMissing(true);

            this.rocksdb = RocksDb.Open(options, dataFolder.ChainPath);
        }
예제 #16
0
        public RocksDbBlockRepository(Network network, string dataFolder, DBreezeSerializer dataStoreSerializer)
        {
            Directory.CreateDirectory(dataFolder);

            this.dbOptions = new DbOptions().SetCreateIfMissing(true);
            this.rocksDb   = RocksDb.Open(this.dbOptions, dataFolder);

            this.dBreezeSerializer   = dataStoreSerializer;
            this.locker              = new object();
            this.logger              = LogManager.GetCurrentClassLogger();
            this.network             = network;
            this.genesisTransactions = network.GetGenesis().Transactions.ToDictionary(k => k.GetHash());
        }
예제 #17
0
        public Cache(bool shouldRestart)
        {
            var cachePath = Environment.GetEnvironmentVariable("CACHE_PATH") ?? "cache";

            if (shouldRestart)
            {
                HandleRestart(cachePath);
            }

            var options = new DbOptions().SetCreateIfMissing(true);

            _rocksDb = RocksDb.Open(options, cachePath);
        }
예제 #18
0
        public RocksDbContext(RocksDbConfig config)
        {
            if (config == null)
            {
                throw new ArgumentNullException(nameof(config));
            }

            // Initialize RocksDB (Connection String is the path to use)
            var options = new DbOptions().SetCreateIfMissing();

            // TODO #358: please avoid sync IO in constructor -> Open connection with the first operation for now
            _rocksDb = RocksDb.Open(options, config.FilePath);
        }
예제 #19
0
        public DbOnTheRocks(string basePath, string dbPath, IDbConfig dbConfig, ILogManager logManager, ColumnFamilies columnFamilies = null, bool deleteOnStart = false)
        {
            RocksDb Open(DbOptions options, string path, ColumnFamilies families)
            {
                return(families == null?RocksDb.Open(options, path) : RocksDb.Open(options, path, families));
            }

            string fullPath = dbPath.GetApplicationResourcePath(basePath);

            _logger = logManager?.GetClassLogger() ?? NullLogger.Instance;
            if (!Directory.Exists(fullPath))
            {
                Directory.CreateDirectory(fullPath);
            }
            else if (deleteOnStart)
            {
                try
                {
                    Directory.Delete(fullPath, true);
                }
                catch (Exception e)
                {
                    if (_logger.IsWarn)
                    {
                        _logger.Warn($"This is not a problem but I could not delete the pending tx database on startup. {e.Message}");
                    }
                }
            }

            try
            {
                // ReSharper disable once VirtualMemberCallInConstructor
                if (_logger.IsDebug)
                {
                    _logger.Debug($"Building options for {Name} DB");
                }
                DbOptions options = BuildOptions(dbConfig);

                // ReSharper disable once VirtualMemberCallInConstructor
                if (_logger.IsInfo)
                {
                    _logger.Info($"Loading {Name.PadRight(16)} from {fullPath} with max memory footprint of {_maxThisDbSize / 1024 / 1024}MB");
                }
                Db = DbsByPath.GetOrAdd(fullPath, path => Open(options, path, columnFamilies));
            }
            catch (DllNotFoundException e) when(e.Message.Contains("libdl"))
            {
                throw new ApplicationException($"Unable to load 'libdl' necessary to init the RocksDB database. Please run{Environment.NewLine}" +
                                               "sudo apt update && sudo apt install libsnappy-dev libc6-dev libc6");
            }
        }
예제 #20
0
파일: Db.cs 프로젝트: ren85/linqdb
 public void Dispose()
 {
     is_disposed = true;
     if (leveld_db != null)
     {
         leveld_db.Dispose();
         leveld_db = null;
     }
     if (struct_db != null)
     {
         struct_db.Dispose();
         struct_db = null;
     }
 }
예제 #21
0
        public void Initialize()
        {
            this.dbOptions = new DbOptions().SetCreateIfMissing(true);
            this.rocksDb   = RocksDb.Open(this.dbOptions, this.dataFolder);

            Block genesis = this.network.GetGenesis();

            if (this.GetTipHash() == null)
            {
                this.SetBlockHash(new HashHeightPair(genesis.GetHash(), 0));
            }

            this.logger.Info("Coinview initialized with tip '{0}'.", this.persistedCoinviewTip);
        }
예제 #22
0
        /// <summary>
        /// Creates a new <seealso cref="RocksDBStore"/>.
        /// </summary>
        /// <param name="path">The path of the directory where the storage files will be saved.
        /// </param>
        /// <param name="blockCacheSize">The capacity of the block cache.</param>
        /// <param name="txCacheSize">The capacity of the transaction cache.</param>
        /// <param name="statesCacheSize">The capacity of the states cache.</param>
        public RocksDBStore(
            string path,
            int blockCacheSize  = 512,
            int txCacheSize     = 1024,
            int statesCacheSize = 10000
            )
        {
            _logger = Log.ForContext <RocksDBStore>();

            if (path is null)
            {
                throw new ArgumentNullException(nameof(path));
            }

            path = Path.GetFullPath(path);

            if (!Directory.Exists(path))
            {
                Directory.CreateDirectory(path);
            }

            _txCache     = new LruCache <TxId, object>(capacity: txCacheSize);
            _blockCache  = new LruCache <HashDigest <SHA256>, BlockDigest>(capacity: blockCacheSize);
            _statesCache = new LruCache <HashDigest <SHA256>, IImmutableDictionary <string, IValue> >(
                capacity: statesCacheSize
                );
            _lastStateRefCaches =
                new Dictionary <Guid, LruCache <string, Tuple <HashDigest <SHA256>, long> > >();

            _path    = path;
            _options = new DbOptions()
                       .SetCreateIfMissing();

            _blockDb    = RocksDBUtils.OpenRocksDb(_options, RocksDbPath(BlockDbName));
            _txDb       = RocksDBUtils.OpenRocksDb(_options, RocksDbPath(TxDbName));
            _stateDb    = RocksDBUtils.OpenRocksDb(_options, RocksDbPath(StateDbName));
            _stagedTxDb = RocksDBUtils.OpenRocksDb(_options, RocksDbPath(StagedTxDbName));

            // When opening a DB in a read-write mode, you need to specify all Column Families that
            // currently exist in a DB. https://github.com/facebook/rocksdb/wiki/Column-Families
            var chainDbColumnFamilies = GetColumnFamilies(_options, ChainDbName);

            _chainDb = RocksDBUtils.OpenRocksDb(
                _options, RocksDbPath(ChainDbName), chainDbColumnFamilies);

            var stateRefDbColumnFamilies = GetColumnFamilies(_options, StateRefDbName);

            _stateRefDb = RocksDBUtils.OpenRocksDb(
                _options, RocksDbPath(StateRefDbName), stateRefDbColumnFamilies);
        }
예제 #23
0
        public PageRepository(IOptions <CachingConfig> cachingConfig,
                              ILogger <CachingConfig> logger)
        {
            _cachingConfig = cachingConfig;
            _logger        = logger;

            var options = new DbOptions()
                          .SetCreateIfMissing(true);

            _dbInstance = RocksDb.OpenWithTtl(options,
                                              Path.Combine(_cachingConfig.Value.Path, "PageCache.db"),
                                              _cachingConfig.Value.CacheTimeToLive);
            _formatter = new BinaryFormatter();
        }
예제 #24
0
        private static bool CheckPwd(string base64Pwd)
        {
            byte[]    pwdBytes = Convert.FromBase64String(base64Pwd);
            string    pwd      = Encoding.UTF8.GetString(pwdBytes);
            DbOptions dbop     = new DbOptions()
                                 .SetCreateIfMissing()
                                 .SetEnableWriteThreadAdaptiveYield(true);
            RocksDb db = RocksDb.Open(dbop, @"E:\db\rocksdb\asset");
            // db里的密码做了sha256
            string password = db.Get("assetPwd");

            db.Dispose();
            return(SecurityHelp.StringSHA256(pwd) == password);
        }
예제 #25
0
        public DataContext(string path = "ether.db")
        {
            _lastTxIdDictionary = new Dictionary <string, long>();

            var options = new DbOptions()
                          .SetCreateIfMissing(true)
                          .SetCreateMissingColumnFamilies(true);

            // Create column families
            IEnumerable <string> cols           = null;
            ColumnFamilies       columnFamilies = new ColumnFamilies();

            try
            {
                cols = RocksDb.ListColumnFamilies(options, path);
                foreach (var col in cols)
                {
                    columnFamilies.Add(col, new ColumnFamilyOptions());
                }
            }
            catch (Exception e)
            {
                // Database not exist nothing todo
            }
            finally
            {
                _db = RocksDb.Open(options, path, columnFamilies);

                // Load column families to the dictionary
                _columnFamiliesDictionary = new Dictionary <string, ColumnFamilyHandle>();
                if (cols != null)
                {
                    foreach (var col in cols)
                    {
                        _columnFamiliesDictionary.Add(col, _db.GetColumnFamily(col));
                    }
                    foreach (var col in cols)
                    {
                        // Load latest transaction Ids
                        if (!col.Contains(':') && col != "default")
                        {
                            var lastTxIdstr = _db.Get("lastTxId", GetColFamily(col + ":lastid"));
                            _lastTxIdDictionary.Add(col,
                                                    string.IsNullOrEmpty(lastTxIdstr) ? 0 : long.Parse(lastTxIdstr));
                        }
                    }
                }
            }
        }
                static ColumnFamilies GetColumnFamilies(string path)
                {
                    if (RocksDb.TryListColumnFamilies(new DbOptions(), path, out var names))
                    {
                        var columnFamilyOptions = new ColumnFamilyOptions();
                        var families            = new ColumnFamilies();
                        foreach (var name in names)
                        {
                            families.Add(name, columnFamilyOptions);
                        }
                        return(families);
                    }

                    return(new ColumnFamilies());
                }
예제 #27
0
        static RocksDb GetRocksDb()
        {
            // try find on network
            var options = new DbOptions()
                          .SetCreateIfMissing(true)
                          .SetCreateMissingColumnFamilies(true);
            var columnFamilies = new ColumnFamilies
            {
                { COLUMN_FAMILY, new ColumnFamilyOptions() }
            };

            var db = RocksDb.Open(options, _path, columnFamilies);

            return(db);
        }
        public static IEnumerable <KeyValuePair <byte[], TValue> > Find <TValue>(this RocksDb db, byte[] keyPrefix, ColumnFamilyHandle?columnFamily = null, ReadOptions?readOptions = null)
            where TValue : ISerializable, new()
        {
            using (var iterator = db.NewIterator(columnFamily, readOptions))
            {
                iterator.Seek(keyPrefix);
                while (iterator.Valid())
                {
                    yield return(new KeyValuePair <byte[], TValue>(
                                     iterator.Key(), iterator.Value().AsSerializable <TValue>()));

                    iterator.Next();
                }
            }
        }
예제 #29
0
        public void ExistAsyncWithExistingBlockReturnsTrue()
        {
            string dir   = CreateTestDir(this);
            Block  block = this.Network.Consensus.ConsensusFactory.CreateBlock();

            using (var engine = RocksDb.Open(new DbOptions().SetCreateIfMissing(true), dir))
            {
                engine.Put(DBH.Key(RocksdbBlockRepository.BlockTableName, block.GetHash().ToBytes()), block.ToBytes());
            }

            using (IBlockRepository repository = this.SetupRepository(this.Network, dir))
            {
                Assert.True(repository.Exist(block.GetHash()));
            }
        }
예제 #30
0
        public void GetTrxAsyncWithoutTransactionIndexReturnsNewTransaction()
        {
            string dir = CreateTestDir(this);

            using (var engine = RocksDb.Open(new DbOptions().SetCreateIfMissing(true), dir))
            {
                engine.Put(DBH.Key(RocksdbBlockRepository.CommonTableName, new byte[0]), this.DataStoreSerializer.Serialize(new HashHeightPair(uint256.Zero, 1)));
                engine.Put(DBH.Key(RocksdbBlockRepository.CommonTableName, new byte[1]), BitConverter.GetBytes(false));
            }

            using (IBlockRepository repository = this.SetupRepository(this.Network, dir))
            {
                Assert.Equal(default(Transaction), repository.GetTransactionById(uint256.Zero));
            }
        }