public RocksDbCacheClient(RpcClient rpcClient, string cachePath)
{
this.rpcClient = rpcClient;
if (!Directory.Exists(cachePath))
{
Directory.CreateDirectory(cachePath);
}
var columnFamilies = GetColumnFamilies(cachePath);
db = RocksDb.Open(new DbOptions().SetCreateIfMissing(true), cachePath, columnFamilies);
public Snapshot(Store store, RocksDb db)
{
this.store = store;
this.db = db;
this.snapshot = db.CreateSnapshot();
this.batch = new WriteBatch();
options = new ReadOptions();
options.SetFillCache(false);
options.SetSnapshot(snapshot);
}
public DbOnTheRocks(string dbPath, IDbConfig dbConfig, ILogManager logManager = null) // TODO: check column families
{
ILogger logger = logManager?.GetClassLogger();
if (!Directory.Exists(dbPath))
{
Directory.CreateDirectory(dbPath);
}
if (logger != null)
{
if (logger.IsInfo)
{
logger.Info($"Using database directory {dbPath}");
}
}
if (dbPath.EndsWith(StateDbPath))
{
_dbInstance = DbInstance.State;
}
else if (dbPath.EndsWith(BlockInfosDbPath))
{
_dbInstance = DbInstance.BlockInfos;
}
else if (dbPath.EndsWith(BlocksDbPath))
{
_dbInstance = DbInstance.Blocks;
}
else if (dbPath.EndsWith(CodeDbPath))
{
_dbInstance = DbInstance.Code;
}
else if (dbPath.EndsWith(ReceiptsDbPath))
{
_dbInstance = DbInstance.Receipts;
}
else if (dbPath.EndsWith(PendingTxsDbPath))
{
_dbInstance = DbInstance.PendingTxs;
}
else if (dbPath.EndsWith(TraceDbPath))
{
_dbInstance = DbInstance.Trace;
}
else
{
_dbInstance = DbInstance.Other;
}
DbOptions options = BuildOptions(dbConfig, _dbInstance);
_db = DbsByPath.GetOrAdd(dbPath, path => RocksDb.Open(options, path));
}
public RocksDbContext(string path = "ChainLachain")
{
_dbpath = path;
_writeOptions = new WriteOptions();
_writeOptions.DisableWal(0);
_writeOptions.SetSync(true);
var options = new DbOptions().SetCreateIfMissing();
_rocksDb = RocksDb.Open(options, _dbpath);
}
public LocalStateRepository(ILogger <LocalStateRepository> logger,
IOptions <RuntimeDirectory> runtimeDirOptions)
{
_logger = logger;
_runtimeDirectory = runtimeDirOptions;
_formatter = new BinaryFormatter();
var options = new DbOptions().SetCreateIfMissing(true);
_dbInstance = RocksDb.Open(options, Path.Combine(_runtimeDirectory.Value.LocalStateRepositoryDb, "LocalOptions.db"));
}
public RocksDbChainStore(Network network, DataFolder dataFolder, ChainIndexer chainIndexer)
{
this.dataFolder = dataFolder.ChainPath;
this.network = network;
this.ChainIndexer = chainIndexer;
this.headers = new MemoryCountCache <uint256, BlockHeader>(601);
this.locker = new object();
this.dbOptions = new DbOptions().SetCreateIfMissing(true);
this.rocksDb = RocksDb.Open(this.dbOptions, this.dataFolder);
}
public RocksDBLogStore(RaftStateMachine rn, string path)
{
this.stateMachine = rn;
string logdbFile = path + "_logdb.db";
var options = new DbOptions()
.SetCreateIfMissing(true);
db = RocksDb.Open(options, logdbFile);
this.ReloadFromStorage();
}
/// <summary>
/// Lists the column families from <see cref="RocksDbSharp.RocksDb"/> located at <paramref name="rocksdbPath"/>.
/// </summary>
/// <param name="rocksdbPath">The path of <see cref="RocksDbSharp.RocksDb"/> to load.</param>
public void List([Option] string? rocksdbPath = null)
{
rocksdbPath ??= Directory.GetCurrentDirectory();
var options = new DbOptions();
var columnFamilies = RocksDb.ListColumnFamilies(
options,
rocksdbPath);
foreach (var columnFamily in columnFamilies)
{
_inputOutputErrorContainer.Out.WriteLine(columnFamily);
}
}
internal static RocksDb OpenRocksDb(
DbOptions options, string dbPath, ColumnFamilies?columnFamilies = null)
{
if (!Directory.Exists(dbPath))
{
Directory.CreateDirectory(dbPath);
}
return(columnFamilies is null
? RocksDb.Open(options, dbPath)
: RocksDb.Open(options, dbPath, columnFamilies));
}
private ColumnFamilyHandle CreateColumnFamilyIfMissing(RocksDb db, string columnFamily)
{
try
{
return(db.GetColumnFamily(columnFamily));
}
catch
{
var options = new ColumnFamilyOptions();
return(db.CreateColumnFamily(options, columnFamily));
}
}
/// <summary>
/// Materialize a <see cref="InMemoryKeyValueStore"/> with the given name.
/// </summary>
/// <typeparam name="KS">New serializer for <typeparamref name="K"/> type</typeparam>
/// <typeparam name="VS">New serializer for <typeparamref name="V"/> type</typeparam>
/// <param name="storeName">the name of the underlying <see cref="IKTable{K, V}"/> state store; valid characters are ASCII alphanumerics, '.', '_' and '-'.</param>
/// <returns>a new <see cref="InMemory{K, V}"/> instance with the given storeName</returns>
public static RocksDb <K, V> @As <KS, VS>(string storeName)
where KS : ISerDes <K>, new()
where VS : ISerDes <V>, new()
{
var m = new RocksDb <K, V>(storeName, new RocksDbKeyValueBytesStoreSupplier(storeName))
{
KeySerdes = new KS(),
ValueSerdes = new VS()
};
return(m);
}
public void Initialize(string connection, string database)
{
if (String.IsNullOrEmpty(connection))
{
throw new ArgumentNullException(nameof(connection), "No connection / path provided for RocksDB");
}
//Connection = path in rocksDB, but I don't like this - we need to rethink how we could
//Make this signature more generic for all of the varieties of repositories
var options = new DbOptions().SetCreateIfMissing(true);
_rocksDb = RocksDb.Open(options, connection);
}
private bool disposedValue = false; // To detect redundant calls
protected virtual void Dispose(bool disposing)
{
if (!disposedValue)
{
if (disposing)
{
_dbInstance.Dispose();
}
_dbInstance = null;
disposedValue = true;
}
}
private bool disposedValue = false; // To detect redundant calls
protected virtual void Dispose(bool disposing)
{
if (!disposedValue)
{
if (disposing)
{
_preAggDatabase?.Dispose();
}
_preAggDatabase = null;
disposedValue = true;
}
}
public RocksdbChainStore(Network network, DataFolder dataFolder, ChainIndexer chainIndexer)
{
this.network = network;
this.ChainIndexer = chainIndexer;
this.nearTipHeaders = new MemoryCountCache <uint256, BlockHeader>(601);
this.recentHeaders = new MemoryCountCache <uint256, BlockHeader>(100);
this.locker = new object();
// Open a connection to a new DB and create if not found
var options = new DbOptions().SetCreateIfMissing(true);
this.rocksdb = RocksDb.Open(options, dataFolder.ChainPath);
}
public RocksDbBlockRepository(Network network, string dataFolder, DBreezeSerializer dataStoreSerializer)
{
Directory.CreateDirectory(dataFolder);
this.dbOptions = new DbOptions().SetCreateIfMissing(true);
this.rocksDb = RocksDb.Open(this.dbOptions, dataFolder);
this.dBreezeSerializer = dataStoreSerializer;
this.locker = new object();
this.logger = LogManager.GetCurrentClassLogger();
this.network = network;
this.genesisTransactions = network.GetGenesis().Transactions.ToDictionary(k => k.GetHash());
}
public Cache(bool shouldRestart)
{
var cachePath = Environment.GetEnvironmentVariable("CACHE_PATH") ?? "cache";
if (shouldRestart)
{
HandleRestart(cachePath);
}
var options = new DbOptions().SetCreateIfMissing(true);
_rocksDb = RocksDb.Open(options, cachePath);
}
public RocksDbContext(RocksDbConfig config)
{
if (config == null)
{
throw new ArgumentNullException(nameof(config));
}
// Initialize RocksDB (Connection String is the path to use)
var options = new DbOptions().SetCreateIfMissing();
// TODO #358: please avoid sync IO in constructor -> Open connection with the first operation for now
_rocksDb = RocksDb.Open(options, config.FilePath);
}
public DbOnTheRocks(string basePath, string dbPath, IDbConfig dbConfig, ILogManager logManager, ColumnFamilies columnFamilies = null, bool deleteOnStart = false)
{
RocksDb Open(DbOptions options, string path, ColumnFamilies families)
{
return(families == null?RocksDb.Open(options, path) : RocksDb.Open(options, path, families));
}
string fullPath = dbPath.GetApplicationResourcePath(basePath);
_logger = logManager?.GetClassLogger() ?? NullLogger.Instance;
if (!Directory.Exists(fullPath))
{
Directory.CreateDirectory(fullPath);
}
else if (deleteOnStart)
{
try
{
Directory.Delete(fullPath, true);
}
catch (Exception e)
{
if (_logger.IsWarn)
{
_logger.Warn($"This is not a problem but I could not delete the pending tx database on startup. {e.Message}");
}
}
}
try
{
// ReSharper disable once VirtualMemberCallInConstructor
if (_logger.IsDebug)
{
_logger.Debug($"Building options for {Name} DB");
}
DbOptions options = BuildOptions(dbConfig);
// ReSharper disable once VirtualMemberCallInConstructor
if (_logger.IsInfo)
{
_logger.Info($"Loading {Name.PadRight(16)} from {fullPath} with max memory footprint of {_maxThisDbSize / 1024 / 1024}MB");
}
Db = DbsByPath.GetOrAdd(fullPath, path => Open(options, path, columnFamilies));
}
catch (DllNotFoundException e) when(e.Message.Contains("libdl"))
{
throw new ApplicationException($"Unable to load 'libdl' necessary to init the RocksDB database. Please run{Environment.NewLine}" +
"sudo apt update && sudo apt install libsnappy-dev libc6-dev libc6");
}
}
public void Dispose()
{
is_disposed = true;
if (leveld_db != null)
{
leveld_db.Dispose();
leveld_db = null;
}
if (struct_db != null)
{
struct_db.Dispose();
struct_db = null;
}
}
public void Initialize()
{
this.dbOptions = new DbOptions().SetCreateIfMissing(true);
this.rocksDb = RocksDb.Open(this.dbOptions, this.dataFolder);
Block genesis = this.network.GetGenesis();
if (this.GetTipHash() == null)
{
this.SetBlockHash(new HashHeightPair(genesis.GetHash(), 0));
}
this.logger.Info("Coinview initialized with tip '{0}'.", this.persistedCoinviewTip);
}
/// <summary>
/// Creates a new <seealso cref="RocksDBStore"/>.
/// </summary>
/// <param name="path">The path of the directory where the storage files will be saved.
/// </param>
/// <param name="blockCacheSize">The capacity of the block cache.</param>
/// <param name="txCacheSize">The capacity of the transaction cache.</param>
/// <param name="statesCacheSize">The capacity of the states cache.</param>
public RocksDBStore(
string path,
int blockCacheSize = 512,
int txCacheSize = 1024,
int statesCacheSize = 10000
)
{
_logger = Log.ForContext <RocksDBStore>();
if (path is null)
{
throw new ArgumentNullException(nameof(path));
}
path = Path.GetFullPath(path);
if (!Directory.Exists(path))
{
Directory.CreateDirectory(path);
}
_txCache = new LruCache <TxId, object>(capacity: txCacheSize);
_blockCache = new LruCache <HashDigest <SHA256>, BlockDigest>(capacity: blockCacheSize);
_statesCache = new LruCache <HashDigest <SHA256>, IImmutableDictionary <string, IValue> >(
capacity: statesCacheSize
);
_lastStateRefCaches =
new Dictionary <Guid, LruCache <string, Tuple <HashDigest <SHA256>, long> > >();
_path = path;
_options = new DbOptions()
.SetCreateIfMissing();
_blockDb = RocksDBUtils.OpenRocksDb(_options, RocksDbPath(BlockDbName));
_txDb = RocksDBUtils.OpenRocksDb(_options, RocksDbPath(TxDbName));
_stateDb = RocksDBUtils.OpenRocksDb(_options, RocksDbPath(StateDbName));
_stagedTxDb = RocksDBUtils.OpenRocksDb(_options, RocksDbPath(StagedTxDbName));
// When opening a DB in a read-write mode, you need to specify all Column Families that
// currently exist in a DB. https://github.com/facebook/rocksdb/wiki/Column-Families
var chainDbColumnFamilies = GetColumnFamilies(_options, ChainDbName);
_chainDb = RocksDBUtils.OpenRocksDb(
_options, RocksDbPath(ChainDbName), chainDbColumnFamilies);
var stateRefDbColumnFamilies = GetColumnFamilies(_options, StateRefDbName);
_stateRefDb = RocksDBUtils.OpenRocksDb(
_options, RocksDbPath(StateRefDbName), stateRefDbColumnFamilies);
}
public PageRepository(IOptions <CachingConfig> cachingConfig,
ILogger <CachingConfig> logger)
{
_cachingConfig = cachingConfig;
_logger = logger;
var options = new DbOptions()
.SetCreateIfMissing(true);
_dbInstance = RocksDb.OpenWithTtl(options,
Path.Combine(_cachingConfig.Value.Path, "PageCache.db"),
_cachingConfig.Value.CacheTimeToLive);
_formatter = new BinaryFormatter();
}
private static bool CheckPwd(string base64Pwd)
{
byte[] pwdBytes = Convert.FromBase64String(base64Pwd);
string pwd = Encoding.UTF8.GetString(pwdBytes);
DbOptions dbop = new DbOptions()
.SetCreateIfMissing()
.SetEnableWriteThreadAdaptiveYield(true);
RocksDb db = RocksDb.Open(dbop, @"E:\db\rocksdb\asset");
// db里的密码做了sha256
string password = db.Get("assetPwd");
db.Dispose();
return(SecurityHelp.StringSHA256(pwd) == password);
}
public DataContext(string path = "ether.db")
{
_lastTxIdDictionary = new Dictionary <string, long>();
var options = new DbOptions()
.SetCreateIfMissing(true)
.SetCreateMissingColumnFamilies(true);
// Create column families
IEnumerable <string> cols = null;
ColumnFamilies columnFamilies = new ColumnFamilies();
try
{
cols = RocksDb.ListColumnFamilies(options, path);
foreach (var col in cols)
{
columnFamilies.Add(col, new ColumnFamilyOptions());
}
}
catch (Exception e)
{
// Database not exist nothing todo
}
finally
{
_db = RocksDb.Open(options, path, columnFamilies);
// Load column families to the dictionary
_columnFamiliesDictionary = new Dictionary <string, ColumnFamilyHandle>();
if (cols != null)
{
foreach (var col in cols)
{
_columnFamiliesDictionary.Add(col, _db.GetColumnFamily(col));
}
foreach (var col in cols)
{
// Load latest transaction Ids
if (!col.Contains(':') && col != "default")
{
var lastTxIdstr = _db.Get("lastTxId", GetColFamily(col + ":lastid"));
_lastTxIdDictionary.Add(col,
string.IsNullOrEmpty(lastTxIdstr) ? 0 : long.Parse(lastTxIdstr));
}
}
}
}
}
static ColumnFamilies GetColumnFamilies(string path)
{
if (RocksDb.TryListColumnFamilies(new DbOptions(), path, out var names))
{
var columnFamilyOptions = new ColumnFamilyOptions();
var families = new ColumnFamilies();
foreach (var name in names)
{
families.Add(name, columnFamilyOptions);
}
return(families);
}
return(new ColumnFamilies());
}
static RocksDb GetRocksDb()
{
// try find on network
var options = new DbOptions()
.SetCreateIfMissing(true)
.SetCreateMissingColumnFamilies(true);
var columnFamilies = new ColumnFamilies
{
{ COLUMN_FAMILY, new ColumnFamilyOptions() }
};
var db = RocksDb.Open(options, _path, columnFamilies);
return(db);
}
public static IEnumerable <KeyValuePair <byte[], TValue> > Find <TValue>(this RocksDb db, byte[] keyPrefix, ColumnFamilyHandle?columnFamily = null, ReadOptions?readOptions = null)
where TValue : ISerializable, new()
{
using (var iterator = db.NewIterator(columnFamily, readOptions))
{
iterator.Seek(keyPrefix);
while (iterator.Valid())
{
yield return(new KeyValuePair <byte[], TValue>(
iterator.Key(), iterator.Value().AsSerializable <TValue>()));
iterator.Next();
}
}
}
public void ExistAsyncWithExistingBlockReturnsTrue()
{
string dir = CreateTestDir(this);
Block block = this.Network.Consensus.ConsensusFactory.CreateBlock();
using (var engine = RocksDb.Open(new DbOptions().SetCreateIfMissing(true), dir))
{
engine.Put(DBH.Key(RocksdbBlockRepository.BlockTableName, block.GetHash().ToBytes()), block.ToBytes());
}
using (IBlockRepository repository = this.SetupRepository(this.Network, dir))
{
Assert.True(repository.Exist(block.GetHash()));
}
}
public void GetTrxAsyncWithoutTransactionIndexReturnsNewTransaction()
{
string dir = CreateTestDir(this);
using (var engine = RocksDb.Open(new DbOptions().SetCreateIfMissing(true), dir))
{
engine.Put(DBH.Key(RocksdbBlockRepository.CommonTableName, new byte[0]), this.DataStoreSerializer.Serialize(new HashHeightPair(uint256.Zero, 1)));
engine.Put(DBH.Key(RocksdbBlockRepository.CommonTableName, new byte[1]), BitConverter.GetBytes(false));
}
using (IBlockRepository repository = this.SetupRepository(this.Network, dir))
{
Assert.Equal(default(Transaction), repository.GetTransactionById(uint256.Zero));
}
}
