private DbOptions BuildOptions(IDbConfig dbConfig, DbInstance dbInstance)
{
BlockBasedTableOptions tableOptions = new BlockBasedTableOptions();
tableOptions.SetBlockSize(16 * 1024);
tableOptions.SetPinL0FilterAndIndexBlocksInCache(true);
tableOptions.SetCacheIndexAndFilterBlocks(ReadConfig <bool>(dbConfig, dbInstance, nameof(dbConfig.CacheIndexAndFilterBlocks)));
tableOptions.SetFilterPolicy(BloomFilterPolicy.Create(10, true));
tableOptions.SetFormatVersion(2);
ulong blockCacheSize = ReadConfig <ulong>(dbConfig, dbInstance, nameof(dbConfig.BlockCacheSize));
IntPtr cache = Native.Instance.rocksdb_cache_create_lru(new UIntPtr(blockCacheSize));
tableOptions.SetBlockCache(cache);
DbOptions options = new DbOptions();
options.SetCreateIfMissing(true);
options.OptimizeForPointLookup(blockCacheSize); // I guess this should be the one option controlled by the DB size property - bind it to LRU cache size
//options.SetCompression(CompressionTypeEnum.rocksdb_snappy_compression);
//options.SetLevelCompactionDynamicLevelBytes(true);
/*
* Multi-Threaded Compactions
* Compactions are needed to remove multiple copies of the same key that may occur if an application overwrites an existing key. Compactions also process deletions of keys. Compactions may occur in multiple threads if configured appropriately.
* The entire database is stored in a set of sstfiles. When a memtable is full, its content is written out to a file in Level-0 (L0). RocksDB removes duplicate and overwritten keys in the memtable when it is flushed to a file in L0. Some files are periodically read in and merged to form larger files - this is called compaction.
* The overall write throughput of an LSM database directly depends on the speed at which compactions can occur, especially when the data is stored in fast storage like SSD or RAM. RocksDB may be configured to issue concurrent compaction requests from multiple threads. It is observed that sustained write rates may increase by as much as a factor of 10 with multi-threaded compaction when the database is on SSDs, as compared to single-threaded compactions.
* TKS: Observed 500MB/s compared to ~100MB/s between multithreaded and single thread compactions on my machine (processor count is returning 12 for 6 cores with hyperthreading)
* TKS: CPU goes to insane 30% usage on idle - compacting only app
*/
options.SetMaxBackgroundCompactions(Environment.ProcessorCount);
//options.SetMaxOpenFiles(32);
options.SetWriteBufferSize(ReadConfig <ulong>(dbConfig, dbInstance, nameof(dbConfig.WriteBufferSize)));
options.SetMaxWriteBufferNumber((int)ReadConfig <uint>(dbConfig, dbInstance, nameof(dbConfig.WriteBufferNumber)));
options.SetMinWriteBufferNumberToMerge(2);
options.SetBlockBasedTableFactory(tableOptions);
options.IncreaseParallelism(Environment.ProcessorCount);
// options.SetLevelCompactionDynamicLevelBytes(true); // only switch on on empty DBs
return(options);
}
private DbOptions BuildOptions(IDbConfig dbConfig)
{
long thisDbSize = 0;
BlockBasedTableOptions tableOptions = new BlockBasedTableOptions();
tableOptions.SetBlockSize(16 * 1024);
tableOptions.SetPinL0FilterAndIndexBlocksInCache(true);
tableOptions.SetCacheIndexAndFilterBlocks(ReadConfig <bool>(dbConfig, nameof(dbConfig.CacheIndexAndFilterBlocks)));
tableOptions.SetFilterPolicy(BloomFilterPolicy.Create(10, true));
tableOptions.SetFormatVersion(2);
ulong blockCacheSize = ReadConfig <ulong>(dbConfig, nameof(dbConfig.BlockCacheSize));
thisDbSize += (long)blockCacheSize;
IntPtr cache = Native.Instance.rocksdb_cache_create_lru(new UIntPtr(blockCacheSize));
tableOptions.SetBlockCache(cache);
DbOptions options = new DbOptions();
options.SetCreateIfMissing(true);
options.SetAdviseRandomOnOpen(true);
options.OptimizeForPointLookup(blockCacheSize); // I guess this should be the one option controlled by the DB size property - bind it to LRU cache size
//options.SetCompression(CompressionTypeEnum.rocksdb_snappy_compression);
//options.SetLevelCompactionDynamicLevelBytes(true);
/*
* Multi-Threaded Compactions
* Compactions are needed to remove multiple copies of the same key that may occur if an application overwrites an existing key. Compactions also process deletions of keys. Compactions may occur in multiple threads if configured appropriately.
* The entire database is stored in a set of sstfiles. When a memtable is full, its content is written out to a file in Level-0 (L0). RocksDB removes duplicate and overwritten keys in the memtable when it is flushed to a file in L0. Some files are periodically read in and merged to form larger files - this is called compaction.
* The overall write throughput of an LSM database directly depends on the speed at which compactions can occur, especially when the data is stored in fast storage like SSD or RAM. RocksDB may be configured to issue concurrent compaction requests from multiple threads. It is observed that sustained write rates may increase by as much as a factor of 10 with multi-threaded compaction when the database is on SSDs, as compared to single-threaded compactions.
* TKS: Observed 500MB/s compared to ~100MB/s between multithreaded and single thread compactions on my machine (processor count is returning 12 for 6 cores with hyperthreading)
* TKS: CPU goes to insane 30% usage on idle - compacting only app
*/
options.SetMaxBackgroundCompactions(Environment.ProcessorCount);
//options.SetMaxOpenFiles(32);
ulong writeBufferSize = ReadConfig <ulong>(dbConfig, nameof(dbConfig.WriteBufferSize));
options.SetWriteBufferSize(writeBufferSize);
int writeBufferNumber = (int)ReadConfig <uint>(dbConfig, nameof(dbConfig.WriteBufferNumber));
options.SetMaxWriteBufferNumber(writeBufferNumber);
options.SetMinWriteBufferNumberToMerge(2);
thisDbSize += (long)writeBufferSize * writeBufferNumber;
Interlocked.Add(ref _maxRocksSize, thisDbSize);
_logger.Info($"Expected max memory footprint of {Name} DB is {thisDbSize / 1024 / 1024}MB ({writeBufferNumber} * {writeBufferSize / 1024 / 1024}MB + {blockCacheSize / 1024 / 1024}MB)");
_logger.Info($"Total max DB footprint so far is {_maxRocksSize / 1024 / 1024}MB");
options.SetBlockBasedTableFactory(tableOptions);
options.SetMaxBackgroundFlushes(Environment.ProcessorCount);
options.IncreaseParallelism(Environment.ProcessorCount);
options.SetRecycleLogFileNum(dbConfig.RecycleLogFileNum); // potential optimization for reusing allocated log files
// options.SetLevelCompactionDynamicLevelBytes(true); // only switch on on empty DBs
_writeOptions = new WriteOptions();
_writeOptions.SetSync(dbConfig.WriteAheadLogSync); // potential fix for corruption on hard process termination, may cause performance degradation
return(options);
}
public DbOnTheRocks(string dbPath) // TODO: check column families
{
if (!Directory.Exists(dbPath))
{
Directory.CreateDirectory(dbPath);
}
// options are based mainly from EtheruemJ at the moment
BlockBasedTableOptions tableOptions = new BlockBasedTableOptions();
//tableOptions.SetPinL0FilterAndIndexBlocksInCache(true);
tableOptions.SetBlockSize(16 * 1024);
//tableOptions.SetCacheIndexAndFilterBlocks(true);
tableOptions.SetFilterPolicy(BloomFilterPolicy.Create(10, false));
tableOptions.SetFormatVersion(2);
DbOptions options = new DbOptions();
options.SetCreateIfMissing(true);
options.OptimizeForPointLookup(32);
//options.SetCompression(CompressionTypeEnum.rocksdb_snappy_compression);
//options.SetLevelCompactionDynamicLevelBytes(true);
//options.SetMaxBackgroundCompactions(4);
//options.SetMaxBackgroundFlushes(2);
//options.SetMaxOpenFiles(32);
//options.SetDbWriteBufferSize(1024 * 1024 * 16);
options.SetWriteBufferSize(1024 * 1024 * 16);
options.SetMaxWriteBufferNumber(6);
options.SetMinWriteBufferNumberToMerge(2);
options.SetBlockBasedTableFactory(tableOptions);
//SliceTransform transform = SliceTransform.CreateFixedPrefix(16);
//options.SetPrefixExtractor(transform);
_db = DbsByPath.GetOrAdd(dbPath, path => RocksDb.Open(options, path));
if (dbPath.EndsWith(StateDbPath))
{
_dbInstance = DbInstance.State;
}
else if (dbPath.EndsWith(BlockInfosDbPath))
{
_dbInstance = DbInstance.BlockInfo;
}
else if (dbPath.EndsWith(BlocksDbPath))
{
_dbInstance = DbInstance.Block;
}
else if (dbPath.EndsWith(CodeDbPath))
{
_dbInstance = DbInstance.Code;
}
else if (dbPath.EndsWith(ReceiptsDbPath))
{
_dbInstance = DbInstance.Receipts;
}
else
{
_dbInstance = DbInstance.Other;
}
}