public RocksDBKeyValueStore(string path)
{
var options = new DbOptions()
.SetCreateIfMissing();
_keyValueDb = RocksDBUtils.OpenRocksDb(options, path);
}
/// <inheritdoc/>
public override bool DeleteBlock(HashDigest <SHA256> blockHash)
{
byte[] key = BlockKey(blockHash);
if (!(_blockIndexDb.Get(key) is byte[] blockDbNameByte))
{
return(false);
}
_rwBlockLock.EnterWriteLock();
try
{
string blockDbName = RocksDBStoreBitConverter.GetString(blockDbNameByte);
if (!_blockDbCache.TryGetValue(blockDbName, out RocksDb blockDb))
{
blockDb = RocksDBUtils.OpenRocksDb(_options, BlockDbPath(blockDbName));
_blockDbCache.AddOrUpdate(blockDbName, blockDb);
}
_blockCache.Remove(blockHash);
_blockIndexDb.Remove(key);
blockDb.Remove(key);
return(true);
}
finally
{
_rwBlockLock.ExitWriteLock();
}
}
/// <inheritdoc/>
public override BlockDigest?GetBlockDigest(HashDigest <SHA256> blockHash)
{
if (_blockCache.TryGetValue(blockHash, out BlockDigest cachedDigest))
{
return(cachedDigest);
}
byte[] key = BlockKey(blockHash);
if (!(_blockIndexDb.Get(key) is byte[] blockDbNameBytes))
{
return(null);
}
_rwBlockLock.EnterReadLock();
try
{
string blockDbName = RocksDBStoreBitConverter.GetString(blockDbNameBytes);
if (!_blockDbCache.TryGetValue(blockDbName, out RocksDb blockDb))
{
blockDb = RocksDBUtils.OpenRocksDb(_options, BlockDbPath(blockDbName));
_blockDbCache.AddOrUpdate(blockDbName, blockDb);
}
byte[] blockBytes = blockDb.Get(key);
BlockDigest blockDigest = BlockDigest.Deserialize(blockBytes);
_blockCache.AddOrUpdate(blockHash, blockDigest);
return(blockDigest);
}
finally
{
_rwBlockLock.ExitReadLock();
}
}
/// <inheritdoc/>
public override bool DeleteTransaction(TxId txid)
{
byte[] key = TxKey(txid);
if (!(_txIndexDb.Get(key) is byte[] txDbNameBytes))
{
return(false);
}
_rwTxLock.EnterWriteLock();
try
{
string txDbName = RocksDBStoreBitConverter.GetString(txDbNameBytes);
if (!_txDbCache.TryGetValue(txDbName, out RocksDb txDb))
{
txDb = RocksDBUtils.OpenRocksDb(_options, TxDbPath(txDbName));
_txDbCache.AddOrUpdate(txDbName, txDb);
}
_txCache.Remove(txid);
_txIndexDb.Remove(key);
txDb.Remove(key);
return(true);
}
finally
{
_rwTxLock.ExitWriteLock();
}
}
/// <inheritdoc/>
public override void PutTransaction <T>(Transaction <T> tx)
{
if (_txCache.ContainsKey(tx.Id))
{
return;
}
byte[] key = TxKey(tx.Id);
if (!(_txIndexDb.Get(key) is null))
{
return;
}
long timestamp = tx.Timestamp.ToUnixTimeSeconds();
string txDbName = $"epoch{(int)timestamp / _txEpochUnitSeconds}";
_rwTxLock.EnterWriteLock();
try
{
if (!_txDbCache.TryGetValue(txDbName, out RocksDb txDb))
{
txDb = RocksDBUtils.OpenRocksDb(_options, TxDbPath(txDbName));
_txDbCache.AddOrUpdate(txDbName, txDb);
}
txDb.Put(key, tx.Serialize(true));
_txIndexDb.Put(key, RocksDBStoreBitConverter.GetBytes(txDbName));
_txCache.AddOrUpdate(tx.Id, tx);
}
finally
{
_rwTxLock.ExitWriteLock();
}
}
/// <summary>
/// Creates a new <seealso cref="RocksDBStore"/>.
/// </summary>
/// <param name="path">The path of the directory where the storage files will be saved.
/// </param>
/// <param name="blockCacheSize">The capacity of the block cache.</param>
/// <param name="txCacheSize">The capacity of the transaction cache.</param>
/// <param name="statesCacheSize">The capacity of the states cache.</param>
/// <param name="maxTotalWalSize">The number to configure <c>max_total_wal_size</c> RocksDB
/// option.</param>
/// <param name="keepLogFileNum">The number to configure <c>keep_log_file_num</c> RocksDB
/// option.</param>
/// <param name="maxLogFileSize">The number to configure <c>max_log_file_size</c>
/// RocksDB option.</param>
public MonoRocksDBStore(
string path,
int blockCacheSize = 512,
int txCacheSize = 1024,
ulong?maxTotalWalSize = null,
ulong?keepLogFileNum = null,
ulong?maxLogFileSize = null
)
{
_logger = Log.ForContext <RocksDBStore>();
if (path is null)
{
throw new ArgumentNullException(nameof(path));
}
path = Path.GetFullPath(path);
if (!Directory.Exists(path))
{
Directory.CreateDirectory(path);
}
_txCache = new LruCache <TxId, object>(capacity: txCacheSize);
_blockCache = new LruCache <HashDigest <SHA256>, BlockDigest>(capacity: blockCacheSize);
_path = path;
_options = new DbOptions()
.SetCreateIfMissing();
if (maxTotalWalSize is ulong maxTotalWalSizeValue)
{
_options = _options.SetMaxTotalWalSize(maxTotalWalSizeValue);
}
if (keepLogFileNum is ulong keepLogFileNumValue)
{
_options = _options.SetKeepLogFileNum(keepLogFileNumValue);
}
if (maxLogFileSize is ulong maxLogFileSizeValue)
{
_options = _options.SetMaxLogFileSize(maxLogFileSizeValue);
}
_blockDb = RocksDBUtils.OpenRocksDb(_options, RocksDbPath(BlockDbName));
_blockPerceptionDb =
RocksDBUtils.OpenRocksDb(_options, RocksDbPath(BlockPerceptionDbName));
_txDb = RocksDBUtils.OpenRocksDb(_options, RocksDbPath(TxDbName));
_stagedTxDb = RocksDBUtils.OpenRocksDb(_options, RocksDbPath(StagedTxDbName));
// When opening a DB in a read-write mode, you need to specify all Column Families that
// currently exist in a DB. https://github.com/facebook/rocksdb/wiki/Column-Families
var chainDbColumnFamilies = GetColumnFamilies(_options, ChainDbName);
_chainDb = RocksDBUtils.OpenRocksDb(
_options, RocksDbPath(ChainDbName), chainDbColumnFamilies);
}
/// <summary>
/// Creates a new <seealso cref="RocksDBStore"/>.
/// </summary>
/// <param name="path">The path of the directory where the storage files will be saved.
/// </param>
/// <param name="blockCacheSize">The capacity of the block cache.</param>
/// <param name="txCacheSize">The capacity of the transaction cache.</param>
/// <param name="statesCacheSize">The capacity of the states cache.</param>
public RocksDBStore(
string path,
int blockCacheSize = 512,
int txCacheSize = 1024,
int statesCacheSize = 10000
)
{
_logger = Log.ForContext <RocksDBStore>();
if (path is null)
{
throw new ArgumentNullException(nameof(path));
}
path = Path.GetFullPath(path);
if (!Directory.Exists(path))
{
Directory.CreateDirectory(path);
}
_txCache = new LruCache <TxId, object>(capacity: txCacheSize);
_blockCache = new LruCache <HashDigest <SHA256>, BlockDigest>(capacity: blockCacheSize);
_statesCache = new LruCache <HashDigest <SHA256>, IImmutableDictionary <string, IValue> >(
capacity: statesCacheSize
);
_lastStateRefCaches =
new Dictionary <Guid, LruCache <string, Tuple <HashDigest <SHA256>, long> > >();
_path = path;
_options = new DbOptions()
.SetCreateIfMissing();
_blockDb = RocksDBUtils.OpenRocksDb(_options, RocksDbPath(BlockDbName));
_txDb = RocksDBUtils.OpenRocksDb(_options, RocksDbPath(TxDbName));
_stateDb = RocksDBUtils.OpenRocksDb(_options, RocksDbPath(StateDbName));
_stagedTxDb = RocksDBUtils.OpenRocksDb(_options, RocksDbPath(StagedTxDbName));
// When opening a DB in a read-write mode, you need to specify all Column Families that
// currently exist in a DB. https://github.com/facebook/rocksdb/wiki/Column-Families
var chainDbColumnFamilies = GetColumnFamilies(_options, ChainDbName);
_chainDb = RocksDBUtils.OpenRocksDb(
_options, RocksDbPath(ChainDbName), chainDbColumnFamilies);
var stateRefDbColumnFamilies = GetColumnFamilies(_options, StateRefDbName);
_stateRefDb = RocksDBUtils.OpenRocksDb(
_options, RocksDbPath(StateRefDbName), stateRefDbColumnFamilies);
}
/// <inheritdoc/>
public override void PutBlock <T>(Block <T> block)
{
if (_blockCache.ContainsKey(block.Hash))
{
return;
}
byte[] key = BlockKey(block.Hash);
if (!(_blockIndexDb.Get(key) is null))
{
return;
}
long timestamp = block.Timestamp.ToUnixTimeSeconds();
foreach (Transaction <T> tx in block.Transactions)
{
PutTransaction(tx);
}
_rwBlockLock.EnterWriteLock();
try
{
string blockDbName = $"epoch{timestamp / _blockEpochUnitSeconds}";
RocksDb blockDb;
lock (_blockDbCache)
{
if (!_blockDbCache.TryGetValue(blockDbName, out blockDb))
{
blockDb = RocksDBUtils.OpenRocksDb(_options, BlockDbPath(blockDbName));
_blockDbCache.AddOrUpdate(blockDbName, blockDb);
}
}
BlockDigest digest = BlockDigest.FromBlock(block);
byte[] value = digest.Serialize();
blockDb.Put(key, value);
_blockIndexDb.Put(key, RocksDBStoreBitConverter.GetBytes(blockDbName));
_blockCache.AddOrUpdate(block.Hash, digest);
}
catch (Exception e)
{
LogUnexpectedException(nameof(PutBlock), e);
}
finally
{
_rwBlockLock.ExitWriteLock();
}
}
/// <inheritdoc/>
public override Transaction <T> GetTransaction <T>(TxId txid)
{
if (_txCache.TryGetValue(txid, out object cachedTx))
{
return((Transaction <T>)cachedTx);
}
byte[] key = TxKey(txid);
if (!(_txIndexDb.Get(key) is byte[] txDbNameBytes))
{
return(null);
}
string txDbName = RocksDBStoreBitConverter.GetString(txDbNameBytes);
_rwTxLock.EnterReadLock();
try
{
RocksDb txDb;
lock (_txDbCache)
{
if (!_txDbCache.TryGetValue(txDbName, out txDb))
{
txDb = RocksDBUtils.OpenRocksDb(_options, TxDbPath(txDbName));
_txDbCache.AddOrUpdate(txDbName, txDb);
}
}
byte[] txBytes = txDb.Get(key);
Transaction <T> tx = Transaction <T> .Deserialize(txBytes, false);
_txCache.AddOrUpdate(txid, tx);
return(tx);
}
catch (Exception e)
{
LogUnexpectedException(nameof(GetTransaction), e);
return(null);
}
finally
{
_rwTxLock.ExitReadLock();
}
}
/// <inheritdoc cref="BaseStore.DeleteBlock(BlockHash)"/>
public override bool DeleteBlock(BlockHash blockHash)
{
byte[] key = BlockKey(blockHash);
if (!(_blockIndexDb.Get(key) is byte[] blockDbNameByte))
{
return(false);
}
_rwBlockLock.EnterWriteLock();
try
{
string blockDbName = RocksDBStoreBitConverter.GetString(blockDbNameByte);
RocksDb blockDb;
lock (_blockDbCache)
{
if (!_blockDbCache.TryGetValue(blockDbName, out blockDb))
{
blockDb = RocksDBUtils.OpenRocksDb(_options, BlockDbPath(blockDbName));
_blockDbCache.AddOrUpdate(blockDbName, blockDb);
}
}
_blockCache.Remove(blockHash);
_blockIndexDb.Remove(key);
blockDb.Remove(key);
return(true);
}
catch (Exception e)
{
LogUnexpectedException(nameof(DeleteBlock), e);
}
finally
{
_rwBlockLock.ExitWriteLock();
}
return(false);
}
/// <summary>
/// Creates a new <seealso cref="RocksDBStore"/>.
/// </summary>
/// <param name="path">The path of the directory where the storage files will be saved.
/// </param>
/// <param name="blockCacheSize">The capacity of the block cache.</param>
/// <param name="txCacheSize">The capacity of the transaction cache.</param>
/// <param name="maxTotalWalSize">The number to configure <c>max_total_wal_size</c> RocksDB
/// option.</param>
/// <param name="keepLogFileNum">The number to configure <c>keep_log_file_num</c> RocksDB
/// option.</param>
/// <param name="maxLogFileSize">The number to configure <c>max_log_file_size</c>
/// RocksDB option.</param>
/// <param name="txEpochUnitSeconds">The interval between epochs of DB partions containing
/// transactions. 86,400 seconds by default.</param>
/// <param name="blockEpochUnitSeconds">The interval between epochs of DB partions
/// containing blocks. 86,400 seconds by default.</param>
/// <param name="dbConnectionCacheSize">The capacity of the block and transaction
/// RocksDB connection cache. 100 by default.</param>
public RocksDBStore(
string path,
int blockCacheSize = 512,
int txCacheSize = 1024,
ulong?maxTotalWalSize = null,
ulong?keepLogFileNum = null,
ulong?maxLogFileSize = null,
int txEpochUnitSeconds = 86400,
int blockEpochUnitSeconds = 86400,
int dbConnectionCacheSize = 100
)
{
_logger = Log.ForContext <RocksDBStore>();
if (path is null)
{
throw new ArgumentNullException(nameof(path));
}
path = Path.GetFullPath(path);
if (!Directory.Exists(path))
{
Directory.CreateDirectory(path);
}
_txCache = new LruCache <TxId, object>(capacity: txCacheSize);
_blockCache = new LruCache <BlockHash, BlockDigest>(capacity: blockCacheSize);
_path = path;
_txEpochUnitSeconds = txEpochUnitSeconds > 0
? txEpochUnitSeconds
: throw new ArgumentException(
"It must be greater than 0.",
nameof(txEpochUnitSeconds));
_blockEpochUnitSeconds = blockEpochUnitSeconds > 0
? blockEpochUnitSeconds
: throw new ArgumentException(
"It must be greater than 0.",
nameof(blockEpochUnitSeconds));
_options = new DbOptions()
.SetCreateIfMissing();
if (maxTotalWalSize is ulong maxTotalWalSizeValue)
{
_options = _options.SetMaxTotalWalSize(maxTotalWalSizeValue);
}
if (keepLogFileNum is ulong keepLogFileNumValue)
{
_options = _options.SetKeepLogFileNum(keepLogFileNumValue);
}
if (maxLogFileSize is ulong maxLogFileSizeValue)
{
_options = _options.SetMaxLogFileSize(maxLogFileSizeValue);
}
_blockIndexDb = RocksDBUtils.OpenRocksDb(_options, BlockDbPath(BlockIndexDbName));
_blockPerceptionDb =
RocksDBUtils.OpenRocksDb(_options, RocksDbPath(BlockPerceptionDbName));
_txIndexDb = RocksDBUtils.OpenRocksDb(_options, TxDbPath(TxIndexDbName));
_stagedTxDb = RocksDBUtils.OpenRocksDb(_options, RocksDbPath(StagedTxDbName));
_txExecutionDb =
RocksDBUtils.OpenRocksDb(_options, RocksDbPath(TxExecutionDbName));
_txIdBlockHashIndexDb =
RocksDBUtils.OpenRocksDb(_options, RocksDbPath(TxIdBlockHashIndexDbName));
// When opening a DB in a read-write mode, you need to specify all Column Families that
// currently exist in a DB. https://github.com/facebook/rocksdb/wiki/Column-Families
var chainDbColumnFamilies = GetColumnFamilies(_options, ChainDbName);
_chainDb = RocksDBUtils.OpenRocksDb(
_options, RocksDbPath(ChainDbName), chainDbColumnFamilies);
_rwTxLock = new ReaderWriterLockSlim(LockRecursionPolicy.SupportsRecursion);
_rwBlockLock = new ReaderWriterLockSlim(LockRecursionPolicy.SupportsRecursion);
_blockDbCache = new LruCache <string, RocksDb>(dbConnectionCacheSize);
_blockDbCache.SetPreRemoveDataMethod(db =>
{
db.Dispose();
return(true);
});
_txDbCache = new LruCache <string, RocksDb>(dbConnectionCacheSize);
_txDbCache.SetPreRemoveDataMethod(db =>
{
db.Dispose();
return(true);
});
}
