private static void RocksDb3()
{
string directory = @"C:\Users\Peska\source\repos\Esent.Tests\RocksDB";
var bbto = new BlockBasedTableOptions()
.SetFilterPolicy(BloomFilterPolicy.Create(10, false))
.SetWholeKeyFiltering(false);
var options = new DbOptions()
.SetCreateIfMissing(true)
.SetCreateMissingColumnFamilies(true);
var columnFamilies = new ColumnFamilies
{
{ "default", new ColumnFamilyOptions().OptimizeForPointLookup(256) },
{ "test", new ColumnFamilyOptions()
//.SetWriteBufferSize(writeBufferSize)
//.SetMaxWriteBufferNumber(maxWriteBufferNumber)
//.SetMinWriteBufferNumberToMerge(minWriteBufferNumberToMerge)
.SetMemtableHugePageSize(2 * 1024 * 1024)
.SetPrefixExtractor(SliceTransform.CreateFixedPrefix(8))
.SetBlockBasedTableFactory(bbto) },
};
using (var db = RocksDbSharp.RocksDb.Open(options, directory, columnFamilies))
{
var cf = db.GetColumnFamily("test");
db.Put("00000000Zero", "", cf: cf);
db.Put("00000001Red", "", cf: cf);
db.Put("00000000One", "", cf: cf);
db.Put("00000000Two", "", cf: cf);
db.Put("00000000Three", "", cf: cf);
db.Put("00000001Black", "", cf: cf);
db.Put("00000002Apple", "", cf: cf);
db.Put("00000002Cranberry", "", cf: cf);
db.Put("00000001Green", "", cf: cf);
db.Put("00000002Banana", "", cf: cf);
var readOptions = new ReadOptions().SetIterateUpperBound("00000002");
var iter = db.NewIterator(readOptions: readOptions, cf: cf);
GC.Collect();
GC.WaitForPendingFinalizers();
var b = Encoding.UTF8.GetBytes("00000001");
iter = iter.Seek(b);
while (iter.Valid())
{
Console.WriteLine(iter.StringKey());
iter = iter.Next();
}
Console.ReadKey();
}
}
static LocalMd()
{
string temp = Path.GetTempPath();
_path = Environment.ExpandEnvironmentVariables(Path.Combine(temp, "rocksdb_prefix_example"));
var bbto = new BlockBasedTableOptions()
.SetFilterPolicy(BloomFilterPolicy.Create(10, false))
.SetWholeKeyFiltering(false);
}
static void Main(string[] args)
{
string temp = Path.GetTempPath();
string path = Environment.ExpandEnvironmentVariables(Path.Combine(temp, "rocksdb_prefix_example"));
var bbto = new BlockBasedTableOptions()
.SetFilterPolicy(BloomFilterPolicy.Create(10, false))
.SetWholeKeyFiltering(false)
;
var options = new DbOptions()
.SetCreateIfMissing(true)
.SetCreateMissingColumnFamilies(true)
;
var columnFamilies = new ColumnFamilies
{
{ "default", new ColumnFamilyOptions().OptimizeForPointLookup(256) },
{ "test", new ColumnFamilyOptions()
//.SetWriteBufferSize(writeBufferSize)
//.SetMaxWriteBufferNumber(maxWriteBufferNumber)
//.SetMinWriteBufferNumberToMerge(minWriteBufferNumberToMerge)
.SetMemtableHugePageSize(2 * 1024 * 1024)
.SetPrefixExtractor(SliceTransform.CreateFixedPrefix((ulong)8))
.SetBlockBasedTableFactory(bbto) },
};
using (var db = RocksDb.Open(options, path, columnFamilies))
{
var cf = db.GetColumnFamily("test");
db.Put("00000000Zero", "", cf: cf);
db.Put("00000000One", "", cf: cf);
db.Put("00000000Two", "", cf: cf);
db.Put("00000000Three", "", cf: cf);
db.Put("00000001Red", "", cf: cf);
db.Put("00000001Green", "", cf: cf);
db.Put("00000001Black", "", cf: cf);
db.Put("00000002Apple", "", cf: cf);
db.Put("00000002Cranberry", "", cf: cf);
db.Put("00000002Banana", "", cf: cf);
var readOptions = new ReadOptions();
using (var iter = db.NewIterator(readOptions: readOptions, cf: cf))
{
GC.Collect();
GC.WaitForPendingFinalizers();
var b = Encoding.UTF8.GetBytes("00000001");
iter.Seek(b);
while (iter.Valid())
{
Console.WriteLine(iter.StringKey());
iter.Next();
}
}
}
Console.WriteLine("Done...");
}
private DbOptions BuildOptions(IDbConfig dbConfig)
{
var tableOptions = new BlockBasedTableOptions();
tableOptions.SetBlockSize(16 * 1024);
tableOptions.SetPinL0FilterAndIndexBlocksInCache(true);
tableOptions.SetCacheIndexAndFilterBlocks(ReadConfig <bool>(dbConfig, nameof(dbConfig.CacheIndexAndFilterBlocks)));
tableOptions.SetFilterPolicy(BloomFilterPolicy.Create(10, true));
tableOptions.SetFormatVersion(2);
var blockCacheSize = ReadConfig <ulong>(dbConfig, nameof(dbConfig.BlockCacheSize));
var cache = Native.Instance.rocksdb_cache_create_lru(new UIntPtr(blockCacheSize));
tableOptions.SetBlockCache(cache);
var options = new DbOptions();
options.SetCreateIfMissing(true);
options.SetAdviseRandomOnOpen(true);
options.OptimizeForPointLookup(blockCacheSize); // I guess this should be the one option controlled by the DB size property - bind it to LRU cache size
//options.SetCompression(CompressionTypeEnum.rocksdb_snappy_compression);
//options.SetLevelCompactionDynamicLevelBytes(true);
/*
* Multi-Threaded Compactions
* Compactions are needed to remove multiple copies of the same key that may occur if an application overwrites an existing key. Compactions also process deletions of keys. Compactions may occur in multiple threads if configured appropriately.
* The entire database is stored in a set of sstfiles. When a memtable is full, its content is written out to a file in Level-0 (L0). RocksDB removes duplicate and overwritten keys in the memtable when it is flushed to a file in L0. Some files are periodically read in and merged to form larger files - this is called compaction.
* The overall write throughput of an LSM database directly depends on the speed at which compactions can occur, especially when the data is stored in fast storage like SSD or RAM. RocksDB may be configured to issue concurrent compaction requests from multiple threads. It is observed that sustained write rates may increase by as much as a factor of 10 with multi-threaded compaction when the database is on SSDs, as compared to single-threaded compactions.
* TKS: Observed 500MB/s compared to ~100MB/s between multithreaded and single thread compactions on my machine (processor count is returning 12 for 6 cores with hyperthreading)
* TKS: CPU goes to insane 30% usage on idle - compacting only app
*/
options.SetMaxBackgroundCompactions(Environment.ProcessorCount);
//options.SetMaxOpenFiles(32);
options.SetWriteBufferSize(ReadConfig <ulong>(dbConfig, nameof(dbConfig.WriteBufferSize)));
options.SetMaxWriteBufferNumber((int)ReadConfig <uint>(dbConfig, nameof(dbConfig.WriteBufferNumber)));
options.SetMinWriteBufferNumberToMerge(2);
options.SetBlockBasedTableFactory(tableOptions);
options.SetMaxBackgroundFlushes(Environment.ProcessorCount);
options.IncreaseParallelism(Environment.ProcessorCount);
options.SetRecycleLogFileNum(dbConfig.RecycleLogFileNum); // potential optimization for reusing allocated log files
// options.SetLevelCompactionDynamicLevelBytes(true); // only switch on on empty DBs
_writeOptions = new WriteOptions();
_writeOptions.SetSync(dbConfig.WriteAheadLogSync); // potential fix for corruption on hard process termination, may cause performance degradation
return(options);
}
private static void RocksDb2()
{
string directory = @"C:\Users\Peska\source\repos\Esent.Tests\RocksDB";
DbOptions options = new DbOptions().SetCreateIfMissing();
using (RocksDb rocksDb = RocksDbSharp.RocksDb.Open(options, directory))
{
TestObject o1 = new TestObject {
EQNum = "3L1234", Mnemonic = "20013L1234011"
};
TestObject o2 = new TestObject {
EQNum = "3L5678", Mnemonic = "20023L5678011"
};
TestObject o3 = new TestObject {
EQNum = "3L9012", Mnemonic = "20033L9012011"
};
TestObject o4 = new TestObject {
EQNum = "3L9012", Mnemonic = "20013L9012012"
};
rocksDb.Put(o1.Mnemonic, JsonConvert.SerializeObject(o1));
rocksDb.Put(o2.Mnemonic, JsonConvert.SerializeObject(o2));
rocksDb.Put(o3.Mnemonic, JsonConvert.SerializeObject(o3));
rocksDb.Put(o4.Mnemonic, JsonConvert.SerializeObject(o4));
SliceTransform sliceTransform = SliceTransform.CreateFixedPrefix(4);
BlockBasedTableOptions blockBasedTableOptions = new BlockBasedTableOptions().SetWholeKeyFiltering(false);
ColumnFamilyOptions columnFamilyOptions = new ColumnFamilyOptions()
.SetPrefixExtractor(sliceTransform)
.SetBlockBasedTableFactory(blockBasedTableOptions);
ColumnFamilies columnFamilies = new ColumnFamilies(columnFamilyOptions);
Iterator iterator = rocksDb.NewIterator();
iterator = iterator.Seek("2001");
while (iterator.Valid())
{
string key = iterator.StringKey();
string value = iterator.StringValue();
iterator.Next();
}
}
}
public FullDbOnTheRocks(string dbPath) // TODO: check column families
{
if (!Directory.Exists(dbPath))
{
Directory.CreateDirectory(dbPath);
}
// options are based mainly from EtheruemJ at the moment
BlockBasedTableOptions tableOptions = new BlockBasedTableOptions();
//tableOptions.SetPinL0FilterAndIndexBlocksInCache(true);
tableOptions.SetBlockSize(16 * 1024);
//tableOptions.SetCacheIndexAndFilterBlocks(true);
tableOptions.SetFilterPolicy(BloomFilterPolicy.Create(10, false));
tableOptions.SetFormatVersion(2);
DbOptions options = new DbOptions();
options.SetCreateIfMissing(true);
options.OptimizeForPointLookup(32);
//options.SetCompression(CompressionTypeEnum.rocksdb_snappy_compression);
//options.SetLevelCompactionDynamicLevelBytes(true);
//options.SetMaxBackgroundCompactions(4);
//options.SetMaxBackgroundFlushes(2);
//options.SetMaxOpenFiles(32);
//options.SetDbWriteBufferSize(1024 * 1024 * 16);
options.SetWriteBufferSize(1024 * 1024 * 16);
options.SetMaxWriteBufferNumber(6);
options.SetMinWriteBufferNumberToMerge(2);
options.SetBlockBasedTableFactory(tableOptions);
//SliceTransform transform = SliceTransform.CreateFixedPrefix(16);
//options.SetPrefixExtractor(transform);
_db = DbsByPath.GetOrAdd(dbPath, path => RocksDb.Open(options, path));
if (dbPath.EndsWith(DiscoveryNodesDbPath))
{
_dbInstance = DbInstance.DiscoveryNodes;
}
else if (dbPath.EndsWith(PeersDbPath))
{
_dbInstance = DbInstance.Peers;
}
}
/// <summary>
/// Create rocksdb config and open rocksdb database.
/// </summary>
/// <param name="context"></param>
protected void OpenDatabase(ProcessorContext context)
{
DbOptions dbOptions = new DbOptions();
ColumnFamilyOptions columnFamilyOptions = new ColumnFamilyOptions();
writeOptions = new WriteOptions();
BlockBasedTableOptions tableConfig = new BlockBasedTableOptions();
RocksDbOptions rocksDbOptions = new RocksDbOptions(dbOptions, columnFamilyOptions);
tableConfig.SetBlockCache(RocksDbSharp.Cache.CreateLru(BLOCK_CACHE_SIZE));
tableConfig.SetBlockSize(BLOCK_SIZE);
tableConfig.SetFilterPolicy(BloomFilterPolicy.Create());
rocksDbOptions.SetOptimizeFiltersForHits(1);
rocksDbOptions.SetBlockBasedTableFactory(tableConfig);
rocksDbOptions.SetCompression(COMPRESSION_TYPE);
rocksDbOptions.SetWriteBufferSize(WRITE_BUFFER_SIZE);
rocksDbOptions.SetCompactionStyle(COMPACTION_STYLE);
rocksDbOptions.SetMaxWriteBufferNumber(MAX_WRITE_BUFFERS);
rocksDbOptions.SetCreateIfMissing(true);
rocksDbOptions.SetErrorIfExists(false);
rocksDbOptions.SetInfoLogLevel(RocksLogLevel.ERROR);
// this is the recommended way to increase parallelism in RocksDb
// note that the current implementation of setIncreaseParallelism affects the number
// of compaction threads but not flush threads (the latter remains one). Also
// the parallelism value needs to be at least two because of the code in
// https://github.com/facebook/rocksdb/blob/62ad0a9b19f0be4cefa70b6b32876e764b7f3c11/util/options.cc#L580
// subtracts one from the value passed to determine the number of compaction threads
// (this could be a bug in the RocksDB code and their devs have been contacted).
rocksDbOptions.IncreaseParallelism(Math.Max(Environment.ProcessorCount, 2));
// TODO : wrap writeOptions in rocksDbOptions too
writeOptions.DisableWal(1);
context.Configuration.RocksDbConfigHandler?.Invoke(Name, rocksDbOptions);
rocksDbOptions.SetMinWriteBufferNumberToMerge(2);
DbDir = new DirectoryInfo(Path.Combine(context.StateDir, parentDir, Name));
Directory.CreateDirectory(DbDir.FullName);
OpenRocksDb(dbOptions, columnFamilyOptions);
IsOpen = true;
}
/// <summary>
/// Provides access to and/or creates a RocksDb persistent key-value store.
/// </summary>
/// <param name="storeDirectory">
/// The directory containing the key-value store.
/// </param>
/// <param name="defaultColumnKeyTracked">
/// Whether the default column should be key-tracked.
/// This will create two columns for the same data,
/// one with just keys and the other with key and value.
/// </param>
/// <param name="additionalColumns">
/// The names of any additional column families in the key-value store.
/// If no additional column families are provided, all entries will be stored
/// in the default column.
/// Column families are analogous to tables in relational databases.
/// </param>
/// <param name="additionalKeyTrackedColumns">
/// The names of any additional column families in the key-value store that
/// should also be key-tracked. This will create two columns for the same data,
/// one with just keys and the other with key and value.
/// Column families are analogous to tables in relational databases.
/// </param>
/// <param name="readOnly">
/// Whether the database should be opened read-only. This prevents modifications and
/// creating unnecessary metadata files related to write sessions.
/// </param>
/// <param name="dropMismatchingColumns">
/// If a store already exists at the given directory, whether any columns that mismatch the the columns that were passed into the constructor
/// should be dropped. This will cause data loss and can only be applied in read-write mode.
/// </param>
/// <param name="rotateLogs">
/// Have RocksDb rotate logs, useful for debugging performance issues. It will rotate logs every 12 hours,
/// up to a maximum of 60 logs (i.e. 30 days). When the maximum amount of logs is reached, the oldest logs
/// are overwritten in a circular fashion.
///
/// Every time the RocksDb instance is open, the current log file is truncated, which means that if you
/// open the DB more than once in a 12 hour period, you will only have partial information.
/// </param>
public RocksDbStore(
string storeDirectory,
bool defaultColumnKeyTracked = false,
IEnumerable <string> additionalColumns = null,
IEnumerable <string> additionalKeyTrackedColumns = null,
bool readOnly = false,
bool dropMismatchingColumns = false,
bool rotateLogs = false)
{
m_storeDirectory = storeDirectory;
m_defaults.DbOptions = new DbOptions()
.SetCreateIfMissing(true)
.SetCreateMissingColumnFamilies(true)
// The background compaction threads run in low priority, so they should not hamper the rest of
// the system. The number of cores in the system is what we want here according to official docs,
// and we are setting this to the number of logical processors, which may be higher.
.SetMaxBackgroundCompactions(Environment.ProcessorCount)
.SetMaxBackgroundFlushes(1)
.IncreaseParallelism(Environment.ProcessorCount / 2)
// Ensure we have performance statistics for profiling
.EnableStatistics();
// A small comment on things tested that did not work:
// * SetAllowMmapReads(true) and SetAllowMmapWrites(true) produce a dramatic performance drop
// * SetUseDirectReads(true) disables the OS cache, and although that's good for random point lookups,
// it produces a dramatic performance drop otherwise.
m_defaults.WriteOptions = new WriteOptions()
// Disable the write ahead log to reduce disk IO. The write ahead log
// is used to recover the store on crashes, so a crash will lose some writes.
// Writes will be made in-memory only until the write buffer size
// is reached and then they will be flushed to storage files.
.DisableWal(1)
// This option is off by default, but just making sure that the C# wrapper
// doesn't change anything. The idea is that the DB won't wait for fsync to
// return before acknowledging the write as successful. This affects
// correctness, because a write may be ACKd before it is actually on disk,
// but it is much faster.
.SetSync(false);
var blockBasedTableOptions = new BlockBasedTableOptions()
// Use a bloom filter to help reduce read amplification on point lookups. 10 bits per key yields a
// ~1% false positive rate as per the RocksDB documentation. This builds one filter per SST, which
// means its optimized for not having a key.
.SetFilterPolicy(BloomFilterPolicy.Create(10, false))
// Use a hash index in SST files to speed up point lookup.
.SetIndexType(BlockBasedTableIndexType.HashSearch)
// Whether to use the whole key or a prefix of it (obtained through the prefix extractor below).
// Since the prefix extractor is a no-op, better performance is achieved by turning this off (i.e.
// setting it to true).
.SetWholeKeyFiltering(true);
m_defaults.ColumnFamilyOptions = new ColumnFamilyOptions()
.SetBlockBasedTableFactory(blockBasedTableOptions)
.SetPrefixExtractor(SliceTransform.CreateNoOp());
if (rotateLogs)
{
// Maximum number of information log files
m_defaults.DbOptions.SetKeepLogFileNum(60);
// Do not rotate information logs based on file size
m_defaults.DbOptions.SetMaxLogFileSize(0);
// How long before we rotate the current information log file
m_defaults.DbOptions.SetLogFileTimeToRoll((ulong)TimeSpan.FromHours(12).Seconds);
}
m_columns = new Dictionary <string, ColumnFamilyInfo>();
additionalColumns = additionalColumns ?? CollectionUtilities.EmptyArray <string>();
additionalKeyTrackedColumns = additionalKeyTrackedColumns ?? CollectionUtilities.EmptyArray <string>();
// The columns that exist in the store on disk may not be in sync with the columns being passed into the constructor
HashSet <string> existingColumns;
try
{
existingColumns = new HashSet <string>(RocksDb.ListColumnFamilies(m_defaults.DbOptions, m_storeDirectory));
}
catch (RocksDbException)
{
// If there is no existing store, an exception will be thrown, ignore it
existingColumns = new HashSet <string>();
}
// In read-only mode, open all existing columns in the store without attempting to validate it against the expected column families
if (readOnly)
{
var columnFamilies = new ColumnFamilies();
foreach (var name in existingColumns)
{
columnFamilies.Add(name, m_defaults.ColumnFamilyOptions);
}
m_store = RocksDb.OpenReadOnly(m_defaults.DbOptions, m_storeDirectory, columnFamilies, errIfLogFileExists: false);
}
else
{
// For read-write mode, column families may be added, so set up column families schema
var columnsSchema = new HashSet <string>(additionalColumns);
// Default column
columnsSchema.Add(ColumnFamilies.DefaultName);
// For key-tracked column familiies, create two columns:
// 1: Normal column of { key : value }
// 2: Key-tracking column of { key : empty-value }
if (defaultColumnKeyTracked)
{
// To be robust to the RocksDB-selected default column name changing,
// just name the default column's key-tracking column KeyColumnSuffix
columnsSchema.Add(KeyColumnSuffix);
}
foreach (var name in additionalKeyTrackedColumns)
{
columnsSchema.Add(name);
columnsSchema.Add(name + KeyColumnSuffix);
}
// Figure out which columns are not part of the schema
var outsideSchemaColumns = new List <string>(existingColumns.Except(columnsSchema));
// RocksDB requires all columns in the store to be opened in read-write mode, so merge existing columns
// with the columns schema that was passed into the constructor
existingColumns.UnionWith(columnsSchema);
var columnFamilies = new ColumnFamilies();
foreach (var name in existingColumns)
{
columnFamilies.Add(name, m_defaults.ColumnFamilyOptions);
}
m_store = RocksDb.Open(m_defaults.DbOptions, m_storeDirectory, columnFamilies);
// Provide an opportunity to update the store to the new column family schema
if (dropMismatchingColumns)
{
foreach (var name in outsideSchemaColumns)
{
m_store.DropColumnFamily(name);
existingColumns.Remove(name);
}
}
}
var userFacingColumns = existingColumns.Where(name => !name.EndsWith(KeyColumnSuffix));
foreach (var name in userFacingColumns)
{
var isKeyTracked = existingColumns.Contains(name + KeyColumnSuffix);
m_columns.Add(name, new ColumnFamilyInfo()
{
Handle = m_store.GetColumnFamily(name),
UseKeyTracking = isKeyTracked,
KeyHandle = isKeyTracked ? m_store.GetColumnFamily(name + KeyColumnSuffix) : null,
});
}
m_columns.TryGetValue(ColumnFamilies.DefaultName, out m_defaultColumnFamilyInfo);
}
/// <summary>
/// Provides access to and/or creates a RocksDb persistent key-value store.
/// </summary>
public RocksDbStore(RocksDbStoreArguments arguments)
{
m_storeDirectory = arguments.StoreDirectory;
m_openBulkLoad = arguments.OpenBulkLoad;
m_defaults.DbOptions = new DbOptions()
.SetCreateIfMissing(true)
.SetCreateMissingColumnFamilies(true)
// The background compaction threads run in low priority, so they should not hamper the rest of
// the system. The number of cores in the system is what we want here according to official docs,
// and we are setting this to the number of logical processors, which may be higher.
// See: https://github.com/facebook/rocksdb/wiki/RocksDB-Tuning-Guide#parallelism-options
#if !PLATFORM_OSX
.SetMaxBackgroundCompactions(Environment.ProcessorCount)
.SetMaxBackgroundFlushes(1)
#else
// The memtable uses significant chunks of available system memory on macOS, we increase the number
// of background flushing threads (low priority) and set the DB write buffer size. This allows for
// up to 128 MB in memtables across all column families before we flush to disk.
.SetMaxBackgroundCompactions(Environment.ProcessorCount / 4)
.SetMaxBackgroundFlushes(Environment.ProcessorCount / 4)
.SetDbWriteBufferSize(128 << 20)
#endif
.IncreaseParallelism(Environment.ProcessorCount / 2);
if (arguments.EnableStatistics)
{
m_defaults.DbOptions.EnableStatistics();
}
if (arguments.OpenBulkLoad)
{
m_defaults.DbOptions.PrepareForBulkLoad();
}
// Maximum number of information log files
if (arguments.RotateLogsNumFiles != null)
{
m_defaults.DbOptions.SetKeepLogFileNum(arguments.RotateLogsNumFiles.Value);
}
// Do not rotate information logs based on file size
if (arguments.RotateLogsMaxFileSizeBytes != null)
{
m_defaults.DbOptions.SetMaxLogFileSize(arguments.RotateLogsMaxFileSizeBytes.Value);
}
// How long before we rotate the current information log file
if (arguments.RotateLogsMaxAge != null)
{
m_defaults.DbOptions.SetLogFileTimeToRoll((ulong)arguments.RotateLogsMaxAge.Value.Seconds);
}
if (arguments.FastOpen)
{
// max_file_opening_threads is defaulted to 16, so no need to update here.
RocksDbSharp.Native.Instance.rocksdb_options_set_skip_stats_update_on_db_open(m_defaults.DbOptions.Handle, true);
}
if (arguments.DisableAutomaticCompactions)
{
m_defaults.DbOptions.SetDisableAutoCompactions(1);
}
// A small comment on things tested that did not work:
// * SetAllowMmapReads(true) and SetAllowMmapWrites(true) produce a dramatic performance drop
// * SetUseDirectReads(true) disables the OS cache, and although that's good for random point lookups,
// it produces a dramatic performance drop otherwise.
m_defaults.WriteOptions = new WriteOptions()
// Disable the write ahead log to reduce disk IO. The write ahead log
// is used to recover the store on crashes, so a crash will lose some writes.
// Writes will be made in-memory only until the write buffer size
// is reached and then they will be flushed to storage files.
.DisableWal(1)
// This option is off by default, but just making sure that the C# wrapper
// doesn't change anything. The idea is that the DB won't wait for fsync to
// return before acknowledging the write as successful. This affects
// correctness, because a write may be ACKd before it is actually on disk,
// but it is much faster.
.SetSync(false);
var blockBasedTableOptions = new BlockBasedTableOptions()
// Use a bloom filter to help reduce read amplification on point lookups. 10 bits per key yields a
// ~1% false positive rate as per the RocksDB documentation. This builds one filter per SST, which
// means its optimized for not having a key.
.SetFilterPolicy(BloomFilterPolicy.Create(10, false))
// Use a hash index in SST files to speed up point lookup.
.SetIndexType(BlockBasedTableIndexType.HashSearch)
// Whether to use the whole key or a prefix of it (obtained through the prefix extractor below).
// Since the prefix extractor is a no-op, better performance is achieved by turning this off (i.e.
// setting it to true).
.SetWholeKeyFiltering(true);
m_defaults.ColumnFamilyOptions = new ColumnFamilyOptions()
#if PLATFORM_OSX
// As advised by the official documentation, LZ4 is the preferred compression algorithm, our RocksDB
// dynamic library has been compiled to support this on macOS. Fallback to Snappy on other systems (default).
.SetCompression(CompressionTypeEnum.rocksdb_lz4_compression)
#endif
.SetBlockBasedTableFactory(blockBasedTableOptions)
.SetPrefixExtractor(SliceTransform.CreateNoOp());
m_columns = new Dictionary <string, ColumnFamilyInfo>();
// The columns that exist in the store on disk may not be in sync with the columns being passed into the constructor
HashSet <string> existingColumns;
try
{
existingColumns = new HashSet <string>(RocksDb.ListColumnFamilies(m_defaults.DbOptions, m_storeDirectory));
}
catch (RocksDbException)
{
// If there is no existing store, an exception will be thrown, ignore it
existingColumns = new HashSet <string>();
}
// In read-only mode, open all existing columns in the store without attempting to validate it against the expected column families
if (arguments.ReadOnly)
{
var columnFamilies = new ColumnFamilies();
foreach (var name in existingColumns)
{
columnFamilies.Add(name, m_defaults.ColumnFamilyOptions);
}
m_store = RocksDb.OpenReadOnly(m_defaults.DbOptions, m_storeDirectory, columnFamilies, errIfLogFileExists: false);
}
else
{
// For read-write mode, column families may be added, so set up column families schema
var additionalColumns = arguments.AdditionalColumns ?? CollectionUtilities.EmptyArray <string>();
var columnsSchema = new HashSet <string>(additionalColumns);
// Default column
columnsSchema.Add(ColumnFamilies.DefaultName);
// For key-tracked column familiies, create two columns:
// 1: Normal column of { key : value }
// 2: Key-tracking column of { key : empty-value }
if (arguments.DefaultColumnKeyTracked)
{
// To be robust to the RocksDB-selected default column name changing,
// just name the default column's key-tracking column KeyColumnSuffix
columnsSchema.Add(KeyColumnSuffix);
}
var additionalKeyTrackedColumns = arguments.AdditionalKeyTrackedColumns ?? CollectionUtilities.EmptyArray <string>();
foreach (var name in additionalKeyTrackedColumns)
{
columnsSchema.Add(name);
columnsSchema.Add(name + KeyColumnSuffix);
}
// Figure out which columns are not part of the schema
var outsideSchemaColumns = new List <string>(existingColumns.Except(columnsSchema));
// RocksDB requires all columns in the store to be opened in read-write mode, so merge existing columns
// with the columns schema that was passed into the constructor
existingColumns.UnionWith(columnsSchema);
var columnFamilies = new ColumnFamilies();
foreach (var name in existingColumns)
{
columnFamilies.Add(name, m_defaults.ColumnFamilyOptions);
}
m_store = RocksDb.Open(m_defaults.DbOptions, m_storeDirectory, columnFamilies);
// Provide an opportunity to update the store to the new column family schema
if (arguments.DropMismatchingColumns)
{
foreach (var name in outsideSchemaColumns)
{
m_store.DropColumnFamily(name);
existingColumns.Remove(name);
}
}
}
var userFacingColumns = existingColumns.Where(name => !name.EndsWith(KeyColumnSuffix));
foreach (var name in userFacingColumns)
{
var isKeyTracked = existingColumns.Contains(name + KeyColumnSuffix);
m_columns.Add(name, new ColumnFamilyInfo()
{
Handle = m_store.GetColumnFamily(name),
UseKeyTracking = isKeyTracked,
KeyHandle = isKeyTracked ? m_store.GetColumnFamily(name + KeyColumnSuffix) : null,
});
}
m_columns.TryGetValue(ColumnFamilies.DefaultName, out m_defaultColumnFamilyInfo);
}
