static void Main(string[] args)
{
var dbOptions = new DbOptions().SetCreateIfMissing().SetCreateMissingColumnFamilies();
var columnFamilies = new ColumnFamilies
{
{ "default", new ColumnFamilyOptions().OptimizeForPointLookup(256) },
{
"user", new ColumnFamilyOptions()
.SetMemtableHugePageSize(2 * 1024 * 1024)
.SetPrefixExtractor(SliceTransform.CreateFixedPrefix(5))
.SetBlockBasedTableFactory(
new BlockBasedTableOptions().SetFilterPolicy(BloomFilterPolicy.Create(10, false))
.SetWholeKeyFiltering(false))
}
};
var db = RocksDb.Open(dbOptions, "my.db", columnFamilies);
var testFamily = db.GetColumnFamily("user");
db.Put("hello:hi", "hi");
for (var i = 0; i < 5; i++)
{
db.Put($"user:{Guid.NewGuid()}", Guid.NewGuid().ToString(), testFamily);
}
using (var it = db.NewIterator(testFamily))
{
it.Seek(Encoding.UTF8.GetBytes("user:"******"Done!" + db.Get("hello:hi"));
}
internal static RocksDbSharp.RocksDb GetDatabase(string databasePath, SliceTransform prefixTransform)
{
var columnFamilies = new ColumnFamilies();
if (File.Exists(databasePath) || Directory.Exists(databasePath))
{
var currentColumnFamilies = RocksDbSharp.RocksDb.ListColumnFamilies(new DbOptions(), databasePath);
foreach (var columnFamily in currentColumnFamilies.ToHashSet())
{
columnFamilies.Add(columnFamily,
new ColumnFamilyOptions()
.SetPrefixExtractor(prefixTransform)
.SetBlockBasedTableFactory(new BlockBasedTableOptions()
.SetWholeKeyFiltering(true)
.SetIndexType(BlockBasedTableIndexType.Binary)));
}
}
return(RocksDbSharp.RocksDb.Open(new DbOptions()
.SetCreateIfMissing()
.SetCreateMissingColumnFamilies()
.IncreaseParallelism(10),
databasePath,
columnFamilies));
}
private static void RocksDb3()
{
string directory = @"C:\Users\Peska\source\repos\Esent.Tests\RocksDB";
var bbto = new BlockBasedTableOptions()
.SetFilterPolicy(BloomFilterPolicy.Create(10, false))
.SetWholeKeyFiltering(false);
var options = new DbOptions()
.SetCreateIfMissing(true)
.SetCreateMissingColumnFamilies(true);
var columnFamilies = new ColumnFamilies
{
{ "default", new ColumnFamilyOptions().OptimizeForPointLookup(256) },
{ "test", new ColumnFamilyOptions()
//.SetWriteBufferSize(writeBufferSize)
//.SetMaxWriteBufferNumber(maxWriteBufferNumber)
//.SetMinWriteBufferNumberToMerge(minWriteBufferNumberToMerge)
.SetMemtableHugePageSize(2 * 1024 * 1024)
.SetPrefixExtractor(SliceTransform.CreateFixedPrefix(8))
.SetBlockBasedTableFactory(bbto) },
};
using (var db = RocksDbSharp.RocksDb.Open(options, directory, columnFamilies))
{
var cf = db.GetColumnFamily("test");
db.Put("00000000Zero", "", cf: cf);
db.Put("00000001Red", "", cf: cf);
db.Put("00000000One", "", cf: cf);
db.Put("00000000Two", "", cf: cf);
db.Put("00000000Three", "", cf: cf);
db.Put("00000001Black", "", cf: cf);
db.Put("00000002Apple", "", cf: cf);
db.Put("00000002Cranberry", "", cf: cf);
db.Put("00000001Green", "", cf: cf);
db.Put("00000002Banana", "", cf: cf);
var readOptions = new ReadOptions().SetIterateUpperBound("00000002");
var iter = db.NewIterator(readOptions: readOptions, cf: cf);
GC.Collect();
GC.WaitForPendingFinalizers();
var b = Encoding.UTF8.GetBytes("00000001");
iter = iter.Seek(b);
while (iter.Valid())
{
Console.WriteLine(iter.StringKey());
iter = iter.Next();
}
Console.ReadKey();
}
}
static void Main(string[] args)
{
string temp = Path.GetTempPath();
string path = Environment.ExpandEnvironmentVariables(Path.Combine(temp, "rocksdb_prefix_example"));
var bbto = new BlockBasedTableOptions()
.SetFilterPolicy(BloomFilterPolicy.Create(10, false))
.SetWholeKeyFiltering(false)
;
var options = new DbOptions()
.SetCreateIfMissing(true)
.SetCreateMissingColumnFamilies(true)
;
var columnFamilies = new ColumnFamilies
{
{ "default", new ColumnFamilyOptions().OptimizeForPointLookup(256) },
{ "test", new ColumnFamilyOptions()
//.SetWriteBufferSize(writeBufferSize)
//.SetMaxWriteBufferNumber(maxWriteBufferNumber)
//.SetMinWriteBufferNumberToMerge(minWriteBufferNumberToMerge)
.SetMemtableHugePageSize(2 * 1024 * 1024)
.SetPrefixExtractor(SliceTransform.CreateFixedPrefix((ulong)8))
.SetBlockBasedTableFactory(bbto) },
};
using (var db = RocksDb.Open(options, path, columnFamilies))
{
var cf = db.GetColumnFamily("test");
db.Put("00000000Zero", "", cf: cf);
db.Put("00000000One", "", cf: cf);
db.Put("00000000Two", "", cf: cf);
db.Put("00000000Three", "", cf: cf);
db.Put("00000001Red", "", cf: cf);
db.Put("00000001Green", "", cf: cf);
db.Put("00000001Black", "", cf: cf);
db.Put("00000002Apple", "", cf: cf);
db.Put("00000002Cranberry", "", cf: cf);
db.Put("00000002Banana", "", cf: cf);
var readOptions = new ReadOptions();
using (var iter = db.NewIterator(readOptions: readOptions, cf: cf))
{
GC.Collect();
GC.WaitForPendingFinalizers();
var b = Encoding.UTF8.GetBytes("00000001");
iter.Seek(b);
while (iter.Valid())
{
Console.WriteLine(iter.StringKey());
iter.Next();
}
}
}
Console.WriteLine("Done...");
}
private static void RocksDb2()
{
string directory = @"C:\Users\Peska\source\repos\Esent.Tests\RocksDB";
DbOptions options = new DbOptions().SetCreateIfMissing();
using (RocksDb rocksDb = RocksDbSharp.RocksDb.Open(options, directory))
{
TestObject o1 = new TestObject {
EQNum = "3L1234", Mnemonic = "20013L1234011"
};
TestObject o2 = new TestObject {
EQNum = "3L5678", Mnemonic = "20023L5678011"
};
TestObject o3 = new TestObject {
EQNum = "3L9012", Mnemonic = "20033L9012011"
};
TestObject o4 = new TestObject {
EQNum = "3L9012", Mnemonic = "20013L9012012"
};
rocksDb.Put(o1.Mnemonic, JsonConvert.SerializeObject(o1));
rocksDb.Put(o2.Mnemonic, JsonConvert.SerializeObject(o2));
rocksDb.Put(o3.Mnemonic, JsonConvert.SerializeObject(o3));
rocksDb.Put(o4.Mnemonic, JsonConvert.SerializeObject(o4));
SliceTransform sliceTransform = SliceTransform.CreateFixedPrefix(4);
BlockBasedTableOptions blockBasedTableOptions = new BlockBasedTableOptions().SetWholeKeyFiltering(false);
ColumnFamilyOptions columnFamilyOptions = new ColumnFamilyOptions()
.SetPrefixExtractor(sliceTransform)
.SetBlockBasedTableFactory(blockBasedTableOptions);
ColumnFamilies columnFamilies = new ColumnFamilies(columnFamilyOptions);
Iterator iterator = rocksDb.NewIterator();
iterator = iterator.Seek("2001");
while (iterator.Valid())
{
string key = iterator.StringKey();
string value = iterator.StringValue();
iterator.Next();
}
}
}
/// <summary>
/// Provides access to and/or creates a RocksDb persistent key-value store.
/// </summary>
/// <param name="storeDirectory">
/// The directory containing the key-value store.
/// </param>
/// <param name="defaultColumnKeyTracked">
/// Whether the default column should be key-tracked.
/// This will create two columns for the same data,
/// one with just keys and the other with key and value.
/// </param>
/// <param name="additionalColumns">
/// The names of any additional column families in the key-value store.
/// If no additional column families are provided, all entries will be stored
/// in the default column.
/// Column families are analogous to tables in relational databases.
/// </param>
/// <param name="additionalKeyTrackedColumns">
/// The names of any additional column families in the key-value store that
/// should also be key-tracked. This will create two columns for the same data,
/// one with just keys and the other with key and value.
/// Column families are analogous to tables in relational databases.
/// </param>
/// <param name="readOnly">
/// Whether the database should be opened read-only. This prevents modifications and
/// creating unnecessary metadata files related to write sessions.
/// </param>
/// <param name="dropMismatchingColumns">
/// If a store already exists at the given directory, whether any columns that mismatch the the columns that were passed into the constructor
/// should be dropped. This will cause data loss and can only be applied in read-write mode.
/// </param>
/// <param name="rotateLogs">
/// Have RocksDb rotate logs, useful for debugging performance issues. It will rotate logs every 12 hours,
/// up to a maximum of 60 logs (i.e. 30 days). When the maximum amount of logs is reached, the oldest logs
/// are overwritten in a circular fashion.
///
/// Every time the RocksDb instance is open, the current log file is truncated, which means that if you
/// open the DB more than once in a 12 hour period, you will only have partial information.
/// </param>
public RocksDbStore(
string storeDirectory,
bool defaultColumnKeyTracked = false,
IEnumerable <string> additionalColumns = null,
IEnumerable <string> additionalKeyTrackedColumns = null,
bool readOnly = false,
bool dropMismatchingColumns = false,
bool rotateLogs = false)
{
m_storeDirectory = storeDirectory;
m_defaults.DbOptions = new DbOptions()
.SetCreateIfMissing(true)
.SetCreateMissingColumnFamilies(true)
// The background compaction threads run in low priority, so they should not hamper the rest of
// the system. The number of cores in the system is what we want here according to official docs,
// and we are setting this to the number of logical processors, which may be higher.
.SetMaxBackgroundCompactions(Environment.ProcessorCount)
.SetMaxBackgroundFlushes(1)
.IncreaseParallelism(Environment.ProcessorCount / 2)
// Ensure we have performance statistics for profiling
.EnableStatistics();
// A small comment on things tested that did not work:
// * SetAllowMmapReads(true) and SetAllowMmapWrites(true) produce a dramatic performance drop
// * SetUseDirectReads(true) disables the OS cache, and although that's good for random point lookups,
// it produces a dramatic performance drop otherwise.
m_defaults.WriteOptions = new WriteOptions()
// Disable the write ahead log to reduce disk IO. The write ahead log
// is used to recover the store on crashes, so a crash will lose some writes.
// Writes will be made in-memory only until the write buffer size
// is reached and then they will be flushed to storage files.
.DisableWal(1)
// This option is off by default, but just making sure that the C# wrapper
// doesn't change anything. The idea is that the DB won't wait for fsync to
// return before acknowledging the write as successful. This affects
// correctness, because a write may be ACKd before it is actually on disk,
// but it is much faster.
.SetSync(false);
var blockBasedTableOptions = new BlockBasedTableOptions()
// Use a bloom filter to help reduce read amplification on point lookups. 10 bits per key yields a
// ~1% false positive rate as per the RocksDB documentation. This builds one filter per SST, which
// means its optimized for not having a key.
.SetFilterPolicy(BloomFilterPolicy.Create(10, false))
// Use a hash index in SST files to speed up point lookup.
.SetIndexType(BlockBasedTableIndexType.HashSearch)
// Whether to use the whole key or a prefix of it (obtained through the prefix extractor below).
// Since the prefix extractor is a no-op, better performance is achieved by turning this off (i.e.
// setting it to true).
.SetWholeKeyFiltering(true);
m_defaults.ColumnFamilyOptions = new ColumnFamilyOptions()
.SetBlockBasedTableFactory(blockBasedTableOptions)
.SetPrefixExtractor(SliceTransform.CreateNoOp());
if (rotateLogs)
{
// Maximum number of information log files
m_defaults.DbOptions.SetKeepLogFileNum(60);
// Do not rotate information logs based on file size
m_defaults.DbOptions.SetMaxLogFileSize(0);
// How long before we rotate the current information log file
m_defaults.DbOptions.SetLogFileTimeToRoll((ulong)TimeSpan.FromHours(12).Seconds);
}
m_columns = new Dictionary <string, ColumnFamilyInfo>();
additionalColumns = additionalColumns ?? CollectionUtilities.EmptyArray <string>();
additionalKeyTrackedColumns = additionalKeyTrackedColumns ?? CollectionUtilities.EmptyArray <string>();
// The columns that exist in the store on disk may not be in sync with the columns being passed into the constructor
HashSet <string> existingColumns;
try
{
existingColumns = new HashSet <string>(RocksDb.ListColumnFamilies(m_defaults.DbOptions, m_storeDirectory));
}
catch (RocksDbException)
{
// If there is no existing store, an exception will be thrown, ignore it
existingColumns = new HashSet <string>();
}
// In read-only mode, open all existing columns in the store without attempting to validate it against the expected column families
if (readOnly)
{
var columnFamilies = new ColumnFamilies();
foreach (var name in existingColumns)
{
columnFamilies.Add(name, m_defaults.ColumnFamilyOptions);
}
m_store = RocksDb.OpenReadOnly(m_defaults.DbOptions, m_storeDirectory, columnFamilies, errIfLogFileExists: false);
}
else
{
// For read-write mode, column families may be added, so set up column families schema
var columnsSchema = new HashSet <string>(additionalColumns);
// Default column
columnsSchema.Add(ColumnFamilies.DefaultName);
// For key-tracked column familiies, create two columns:
// 1: Normal column of { key : value }
// 2: Key-tracking column of { key : empty-value }
if (defaultColumnKeyTracked)
{
// To be robust to the RocksDB-selected default column name changing,
// just name the default column's key-tracking column KeyColumnSuffix
columnsSchema.Add(KeyColumnSuffix);
}
foreach (var name in additionalKeyTrackedColumns)
{
columnsSchema.Add(name);
columnsSchema.Add(name + KeyColumnSuffix);
}
// Figure out which columns are not part of the schema
var outsideSchemaColumns = new List <string>(existingColumns.Except(columnsSchema));
// RocksDB requires all columns in the store to be opened in read-write mode, so merge existing columns
// with the columns schema that was passed into the constructor
existingColumns.UnionWith(columnsSchema);
var columnFamilies = new ColumnFamilies();
foreach (var name in existingColumns)
{
columnFamilies.Add(name, m_defaults.ColumnFamilyOptions);
}
m_store = RocksDb.Open(m_defaults.DbOptions, m_storeDirectory, columnFamilies);
// Provide an opportunity to update the store to the new column family schema
if (dropMismatchingColumns)
{
foreach (var name in outsideSchemaColumns)
{
m_store.DropColumnFamily(name);
existingColumns.Remove(name);
}
}
}
var userFacingColumns = existingColumns.Where(name => !name.EndsWith(KeyColumnSuffix));
foreach (var name in userFacingColumns)
{
var isKeyTracked = existingColumns.Contains(name + KeyColumnSuffix);
m_columns.Add(name, new ColumnFamilyInfo()
{
Handle = m_store.GetColumnFamily(name),
UseKeyTracking = isKeyTracked,
KeyHandle = isKeyTracked ? m_store.GetColumnFamily(name + KeyColumnSuffix) : null,
});
}
m_columns.TryGetValue(ColumnFamilies.DefaultName, out m_defaultColumnFamilyInfo);
}
/// <summary>
/// Provides access to and/or creates a RocksDb persistent key-value store.
/// </summary>
public RocksDbStore(RocksDbStoreArguments arguments)
{
m_storeDirectory = arguments.StoreDirectory;
m_openBulkLoad = arguments.OpenBulkLoad;
m_defaults.DbOptions = new DbOptions()
.SetCreateIfMissing(true)
.SetCreateMissingColumnFamilies(true)
// The background compaction threads run in low priority, so they should not hamper the rest of
// the system. The number of cores in the system is what we want here according to official docs,
// and we are setting this to the number of logical processors, which may be higher.
// See: https://github.com/facebook/rocksdb/wiki/RocksDB-Tuning-Guide#parallelism-options
#if !PLATFORM_OSX
.SetMaxBackgroundCompactions(Environment.ProcessorCount)
.SetMaxBackgroundFlushes(1)
#else
// The memtable uses significant chunks of available system memory on macOS, we increase the number
// of background flushing threads (low priority) and set the DB write buffer size. This allows for
// up to 128 MB in memtables across all column families before we flush to disk.
.SetMaxBackgroundCompactions(Environment.ProcessorCount / 4)
.SetMaxBackgroundFlushes(Environment.ProcessorCount / 4)
.SetDbWriteBufferSize(128 << 20)
#endif
.IncreaseParallelism(Environment.ProcessorCount / 2);
if (arguments.EnableStatistics)
{
m_defaults.DbOptions.EnableStatistics();
}
if (arguments.OpenBulkLoad)
{
m_defaults.DbOptions.PrepareForBulkLoad();
}
// Maximum number of information log files
if (arguments.RotateLogsNumFiles != null)
{
m_defaults.DbOptions.SetKeepLogFileNum(arguments.RotateLogsNumFiles.Value);
}
// Do not rotate information logs based on file size
if (arguments.RotateLogsMaxFileSizeBytes != null)
{
m_defaults.DbOptions.SetMaxLogFileSize(arguments.RotateLogsMaxFileSizeBytes.Value);
}
// How long before we rotate the current information log file
if (arguments.RotateLogsMaxAge != null)
{
m_defaults.DbOptions.SetLogFileTimeToRoll((ulong)arguments.RotateLogsMaxAge.Value.Seconds);
}
if (arguments.FastOpen)
{
// max_file_opening_threads is defaulted to 16, so no need to update here.
RocksDbSharp.Native.Instance.rocksdb_options_set_skip_stats_update_on_db_open(m_defaults.DbOptions.Handle, true);
}
if (arguments.DisableAutomaticCompactions)
{
m_defaults.DbOptions.SetDisableAutoCompactions(1);
}
// A small comment on things tested that did not work:
// * SetAllowMmapReads(true) and SetAllowMmapWrites(true) produce a dramatic performance drop
// * SetUseDirectReads(true) disables the OS cache, and although that's good for random point lookups,
// it produces a dramatic performance drop otherwise.
m_defaults.WriteOptions = new WriteOptions()
// Disable the write ahead log to reduce disk IO. The write ahead log
// is used to recover the store on crashes, so a crash will lose some writes.
// Writes will be made in-memory only until the write buffer size
// is reached and then they will be flushed to storage files.
.DisableWal(1)
// This option is off by default, but just making sure that the C# wrapper
// doesn't change anything. The idea is that the DB won't wait for fsync to
// return before acknowledging the write as successful. This affects
// correctness, because a write may be ACKd before it is actually on disk,
// but it is much faster.
.SetSync(false);
var blockBasedTableOptions = new BlockBasedTableOptions()
// Use a bloom filter to help reduce read amplification on point lookups. 10 bits per key yields a
// ~1% false positive rate as per the RocksDB documentation. This builds one filter per SST, which
// means its optimized for not having a key.
.SetFilterPolicy(BloomFilterPolicy.Create(10, false))
// Use a hash index in SST files to speed up point lookup.
.SetIndexType(BlockBasedTableIndexType.HashSearch)
// Whether to use the whole key or a prefix of it (obtained through the prefix extractor below).
// Since the prefix extractor is a no-op, better performance is achieved by turning this off (i.e.
// setting it to true).
.SetWholeKeyFiltering(true);
m_defaults.ColumnFamilyOptions = new ColumnFamilyOptions()
#if PLATFORM_OSX
// As advised by the official documentation, LZ4 is the preferred compression algorithm, our RocksDB
// dynamic library has been compiled to support this on macOS. Fallback to Snappy on other systems (default).
.SetCompression(CompressionTypeEnum.rocksdb_lz4_compression)
#endif
.SetBlockBasedTableFactory(blockBasedTableOptions)
.SetPrefixExtractor(SliceTransform.CreateNoOp());
m_columns = new Dictionary <string, ColumnFamilyInfo>();
// The columns that exist in the store on disk may not be in sync with the columns being passed into the constructor
HashSet <string> existingColumns;
try
{
existingColumns = new HashSet <string>(RocksDb.ListColumnFamilies(m_defaults.DbOptions, m_storeDirectory));
}
catch (RocksDbException)
{
// If there is no existing store, an exception will be thrown, ignore it
existingColumns = new HashSet <string>();
}
// In read-only mode, open all existing columns in the store without attempting to validate it against the expected column families
if (arguments.ReadOnly)
{
var columnFamilies = new ColumnFamilies();
foreach (var name in existingColumns)
{
columnFamilies.Add(name, m_defaults.ColumnFamilyOptions);
}
m_store = RocksDb.OpenReadOnly(m_defaults.DbOptions, m_storeDirectory, columnFamilies, errIfLogFileExists: false);
}
else
{
// For read-write mode, column families may be added, so set up column families schema
var additionalColumns = arguments.AdditionalColumns ?? CollectionUtilities.EmptyArray <string>();
var columnsSchema = new HashSet <string>(additionalColumns);
// Default column
columnsSchema.Add(ColumnFamilies.DefaultName);
// For key-tracked column familiies, create two columns:
// 1: Normal column of { key : value }
// 2: Key-tracking column of { key : empty-value }
if (arguments.DefaultColumnKeyTracked)
{
// To be robust to the RocksDB-selected default column name changing,
// just name the default column's key-tracking column KeyColumnSuffix
columnsSchema.Add(KeyColumnSuffix);
}
var additionalKeyTrackedColumns = arguments.AdditionalKeyTrackedColumns ?? CollectionUtilities.EmptyArray <string>();
foreach (var name in additionalKeyTrackedColumns)
{
columnsSchema.Add(name);
columnsSchema.Add(name + KeyColumnSuffix);
}
// Figure out which columns are not part of the schema
var outsideSchemaColumns = new List <string>(existingColumns.Except(columnsSchema));
// RocksDB requires all columns in the store to be opened in read-write mode, so merge existing columns
// with the columns schema that was passed into the constructor
existingColumns.UnionWith(columnsSchema);
var columnFamilies = new ColumnFamilies();
foreach (var name in existingColumns)
{
columnFamilies.Add(name, m_defaults.ColumnFamilyOptions);
}
m_store = RocksDb.Open(m_defaults.DbOptions, m_storeDirectory, columnFamilies);
// Provide an opportunity to update the store to the new column family schema
if (arguments.DropMismatchingColumns)
{
foreach (var name in outsideSchemaColumns)
{
m_store.DropColumnFamily(name);
existingColumns.Remove(name);
}
}
}
var userFacingColumns = existingColumns.Where(name => !name.EndsWith(KeyColumnSuffix));
foreach (var name in userFacingColumns)
{
var isKeyTracked = existingColumns.Contains(name + KeyColumnSuffix);
m_columns.Add(name, new ColumnFamilyInfo()
{
Handle = m_store.GetColumnFamily(name),
UseKeyTracking = isKeyTracked,
KeyHandle = isKeyTracked ? m_store.GetColumnFamily(name + KeyColumnSuffix) : null,
});
}
m_columns.TryGetValue(ColumnFamilies.DefaultName, out m_defaultColumnFamilyInfo);
}
public RocksDatabase(string path, ulong maxKeyLength = 100)
{
Path = path;
RocksDb = RocksDbFactory.GetDatabase(path, SliceTransform.CreateFixedPrefix(maxKeyLength));
MaxKeyLength = maxKeyLength;
}
public void Begin()
{
Random rd = new Random();
config = new StreamConfig();
config.ApplicationId = $"RocksDbOptionsTests";
config.UseRandomRocksDbConfigForTest();
config.RocksDbConfigHandler = (name, options) =>
{
options
.EnableStatistics()
.IncreaseParallelism(parallelism)
.OptimizeForPointLookup(blockCacheSizeMb)
.OptimizeLevelStyleCompaction(memTableMemoryBudget)
.OptimizeUniversalStyleCompaction(memtableMemoryBudget2)
.PrepareForBulkLoad()
.SetAccessHintOnCompactionStart(accessHintOnCompactionStart)
.SetAdviseRandomOnOpen(adviseRandomOnOpen)
.SetAllowConcurrentMemtableWrite(allowConcurrentMemtableWrite)
.SetAllowMmapReads(false)
.SetAllowMmapWrites(false)
.SetArenaBlockSize(arenaBlockSize)
.SetBaseBackgroundCompactions(baseBackgroundCompactions)
.SetBloomLocality(bloomLocality)
.SetBytesPerSync(bytesPerSync)
// .SetCompactionFilter(IntPtr.Zero)
// .SetCompactionFilterFactory(IntPtr.Zero)
.SetCompactionReadaheadSize(1200)
.SetCompactionStyle(RocksDbSharp.Compaction.Level)
//.SetComparator(IntPtr.Zero)
.SetCompression(RocksDbSharp.Compression.Lz4)
.SetCompressionOptions(1, 2, 3, 4)
.SetCompressionPerLevel(new[] { RocksDbSharp.Compression.Lz4 }, 1)
.SetCreateIfMissing()
.SetCreateMissingColumnFamilies()
.SetDbLogDir("test")
.SetDbWriteBufferSize(1000)
.SetDeleteObsoleteFilesPeriodMicros(50)
.SetDisableAutoCompactions(1)
.SetEnableWriteThreadAdaptiveYield(true)
//.SetEnv(IntPtr.Zero)
.SetErrorIfExists()
//.SetFifoCompactionOptions(IntPtr.Zero)
.SetHardPendingCompactionBytesLimit(1)
.SetHardRateLimit(40597)
.SetHashLinkListRep(12)
.SetHashSkipListRep(56, 4, 2)
//.SetInfoLog(IntPtr.Zero)
.SetInfoLogLevel(RocksLogLevel.NUM_INFO_LOG)
.SetInplaceUpdateNumLocks(134)
.SetIsFdCloseOnExec(false)
.SetKeepLogFileNum(1)
.SetLevel0FileNumCompactionTrigger(14)
.SetLevel0SlowdownWritesTrigger(144)
.SetInplaceUpdateSupport(true)
.SetLevel0StopWritesTrigger(24)
.SetLevelCompactionDynamicLevelBytes(true)
.SetLogFileTimeToRoll(154)
.SetManifestPreallocationSize(153)
.SetMaxBackgroundFlushes(3)
.SetMaxBytesForLevelBase(1453)
.SetMaxBytesForLevelMultiplier(2)
.SetMaxBytesForLevelMultiplierAdditional(new[] { 1 }, 1)
.SetMaxCompactionBytes(345678)
.SetMaxFileOpeningThreads(2)
.SetMaxLogFileSize(1)
.SetMaxManifestFileSize(131)
.SetMaxMemCompactionLevel(12)
.SetMaxOpenFiles(20)
.SetMaxSequentialSkipInIterations(13)
.SetMaxSuccessiveMerges(12)
.SetMaxTotalWalSize(12)
.SetMaxWriteBufferNumber(15543)
.SetMaxWriteBufferNumberToMaintain(126)
.SetMemtableHugePageSize(64317)
.SetMemtableHugePageSize(64317)
.SetMemtablePrefixBloomSizeRatio(12)
//.SetMergeOperator(IntPtr.Zero)
.SetMinLevelToCompress(1)
.SetMemtableVectorRep()
.SetMinWriteBufferNumberToMerge(1312)
.SetNumLevels(1)
.SetOptimizeFiltersForHits(56)
.SetParanoidChecks()
.SetSkipLogErrorOnRecovery(true)
.SetSoftPendingCompactionBytesLimit(134)
.SetSoftRateLimit(131)
.SetStatsDumpPeriodSec(532)
.SetTableCacheNumShardbits(12)
.SetTableCacheRemoveScanCountLimit(66)
.SetTargetFileSizeBase(6)
.SetTargetFileSizeMultiplier(2)
.SetUint64addMergeOperator()
//.SetUniversalCompactionOptions(IntPtr.Zero)
.SetUseAdaptiveMutex(false)
.SetUseDirectIoForFlushAndCompaction(true)
.SetUseDirectReads(true)
.SetUseFsync(1)
.SetWalRecoveryMode(RocksDbSharp.Recovery.SkipAnyCorruptedRecords)
.SetWALSizeLimitMB(40)
.SetWALTtlSeconds(1454151413)
.SetWriteBufferSize(45678976543)
.SetPrefixExtractor(SliceTransform.CreateNoOp())
.SetPurgeRedundantKvsWhileFlush(false)
.SetRateLimitDelayMaxMilliseconds(1762)
.SetRecycleLogFileNum(1)
.SetReportBgIoStats(true)
.SetPlainTableFactory(1, 23, 4, 2)
.SetMaxBackgroundCompactions(1);
};
id = new TaskId {
Id = 0, Partition = 0
};
partition = new TopicPartition("source", 0);
stateManager = new ProcessorStateManager(
id,
new List <TopicPartition> {
partition
},
null,
new MockChangelogRegister(),
new MockOffsetCheckpointManager());
task = new Mock <AbstractTask>();
task.Setup(k => k.Id).Returns(id);
context = new ProcessorContext(task.Object, config, stateManager, new StreamMetricsRegistry());
store = new RocksDbKeyValueStore("test-store");
store.Init(context, store);
}
protected override void Load(ContainerBuilder builder)
{
// new strategy to let transform author define shorthand
RegisterTransform(builder, (context) => new AbsTransform(context), AbsTransform.GetSignature());
RegisterTransform(builder, (context) => new SliceTransform(context), SliceTransform.GetSignature());
RegisterTransform(builder, (context) => new FormatTransform(context), FormatTransform.GetSignature());
builder.Register((c, p) => new AddTransform(p.Positional <IContext>(0))).Named <ITransform>("add");
builder.Register((c, p) => new EqualsTransform(p.Positional <IContext>(0))).Named <ITransform>("all");
builder.Register((c, p) => new AddTransform(p.Positional <IContext>(0))).Named <ITransform>("sum");
builder.Register((c, p) => new CeilingTransform(p.Positional <IContext>(0))).Named <ITransform>("ceiling");
builder.Register((c, p) => new CoalesceTransform(p.Positional <IContext>(0))).Named <ITransform>("coalesce");
builder.Register((c, p) => new ConcatTransform(p.Positional <IContext>(0))).Named <ITransform>("concat");
builder.Register((c, p) => new ConnectionTransform(p.Positional <IContext>(0))).Named <ITransform>("connection");
builder.Register((c, p) => new ConvertTransform(p.Positional <IContext>(0))).Named <ITransform>("convert");
builder.Register((c, p) => new CopyTransform(p.Positional <IContext>(0))).Named <ITransform>("copy");
builder.Register((c, p) => new DateDiffTransform(p.Positional <IContext>(0))).Named <ITransform>("datediff");
builder.Register((c, p) => new DatePartTransform(p.Positional <IContext>(0))).Named <ITransform>("datepart");
builder.Register((c, p) => new DecompressTransform(p.Positional <IContext>(0))).Named <ITransform>("decompress");
builder.Register((c, p) => new CompressTransform(p.Positional <IContext>(0))).Named <ITransform>("compress");
builder.Register((c, p) => new FileExtTransform(p.Positional <IContext>(0))).Named <ITransform>("fileext");
builder.Register((c, p) => new FileNameTransform(p.Positional <IContext>(0))).Named <ITransform>("filename");
builder.Register((c, p) => new FilePathTransform(p.Positional <IContext>(0))).Named <ITransform>("filepath");
builder.Register((c, p) => new FloorTransform(p.Positional <IContext>(0))).Named <ITransform>("floor");
builder.Register((c, p) => new FormatXmlTransfrom(p.Positional <IContext>(0))).Named <ITransform>("formatxml");
builder.Register((c, p) => new FormatPhoneTransform(p.Positional <IContext>(0))).Named <ITransform>("formatphone");
builder.Register((c, p) => new HashcodeTransform(p.Positional <IContext>(0))).Named <ITransform>("hashcode");
builder.Register((c, p) => new DecodeTransform(p.Positional <IContext>(0))).Named <ITransform>("htmldecode");
builder.Register((c, p) => new HtmlEncodeTransform(p.Positional <IContext>(0))).Named <ITransform>("htmlencode");
builder.Register((c, p) => new InsertTransform(p.Positional <IContext>(0))).Named <ITransform>("insert");
builder.Register((c, p) => new InvertTransform(p.Positional <IContext>(0))).Named <ITransform>("invert");
builder.Register((c, p) => new JoinTransform(p.Positional <IContext>(0))).Named <ITransform>("join");
builder.Register((c, p) => new LastTransform(p.Positional <IContext>(0))).Named <ITransform>("last");
builder.Register((c, p) => new LeftTransform(p.Positional <IContext>(0))).Named <ITransform>("left");
builder.Register((c, p) => new ToLowerTransform(p.Positional <IContext>(0))).Named <ITransform>("lower");
builder.Register((c, p) => new ToLowerTransform(p.Positional <IContext>(0))).Named <ITransform>("tolower");
builder.Register((c, p) => new MapTransform(p.Positional <IContext>(0))).Named <ITransform>("map");
builder.Register((c, p) => new RegexMatchTransform(p.Positional <IContext>(0))).Named <ITransform>("match");
builder.Register((c, p) => new MultiplyTransform(p.Positional <IContext>(0))).Named <ITransform>("multiply");
builder.Register((c, p) => new NextTransform(p.Positional <IContext>(0))).Named <ITransform>("next");
builder.Register((c, p) => new UtcNowTransform(p.Positional <IContext>(0))).Named <ITransform>("now");
builder.Register((c, p) => new PadLeftTransform(p.Positional <IContext>(0))).Named <ITransform>("padleft");
builder.Register((c, p) => new PadRightTransform(p.Positional <IContext>(0))).Named <ITransform>("padright");
builder.Register((c, p) => new RazorTransform(p.Positional <IContext>(0))).Named <ITransform>("razor");
builder.Register((c, p) => new TimeZoneTransform(p.Positional <IContext>(0))).Named <ITransform>("timezone");
builder.Register((c, p) => new RegexReplaceTransform(p.Positional <IContext>(0))).Named <ITransform>("regexreplace");
builder.Register((c, p) => new RemoveTransform(p.Positional <IContext>(0))).Named <ITransform>("remove");
builder.Register((c, p) => new ReplaceTransform(p.Positional <IContext>(0))).Named <ITransform>("replace");
builder.Register((c, p) => new RightTransform(p.Positional <IContext>(0))).Named <ITransform>("right");
builder.Register((c, p) => new RoundTransform(p.Positional <IContext>(0))).Named <ITransform>("round");
builder.Register((c, p) => new RoundToTransform(p.Positional <IContext>(0), RoundTo.Nearest)).Named <ITransform>("roundto");
builder.Register((c, p) => new RoundToTransform(p.Positional <IContext>(0), RoundTo.Up)).Named <ITransform>("roundupto");
builder.Register((c, p) => new RoundToTransform(p.Positional <IContext>(0), RoundTo.Down)).Named <ITransform>("rounddownto");
builder.Register((c, p) => new SplitLengthTransform(p.Positional <IContext>(0))).Named <ITransform>("splitlength");
builder.Register((c, p) => new SubStringTransform(p.Positional <IContext>(0))).Named <ITransform>("substring");
builder.Register((c, p) => new TagTransform(p.Positional <IContext>(0))).Named <ITransform>("tag");
builder.Register((c, p) => new RelativeTimeTransform(p.Positional <IContext>(0), true)).Named <ITransform>("timeago");
builder.Register((c, p) => new RelativeTimeTransform(p.Positional <IContext>(0), false)).Named <ITransform>("timeahead");
builder.Register((c, p) => new ToStringTransform(p.Positional <IContext>(0))).Named <ITransform>("tostring");
builder.Register((c, p) => new ToTimeTransform(p.Positional <IContext>(0))).Named <ITransform>("totime");
builder.Register((c, p) => new ToYesNoTransform(p.Positional <IContext>(0))).Named <ITransform>("toyesno");
builder.Register((c, p) => new TrimTransform(p.Positional <IContext>(0))).Named <ITransform>("trim");
builder.Register((c, p) => new TrimEndTransform(p.Positional <IContext>(0))).Named <ITransform>("trimend");
builder.Register((c, p) => new TrimStartTransform(p.Positional <IContext>(0))).Named <ITransform>("trimstart");
builder.Register((c, p) => new VelocityTransform(p.Positional <IContext>(0), c.Resolve <IReader>())).Named <ITransform>("velocity");
builder.Register((c, p) => new ToUpperTransform(p.Positional <IContext>(0))).Named <ITransform>("upper");
builder.Register((c, p) => new ToUpperTransform(p.Positional <IContext>(0))).Named <ITransform>("toupper");
builder.Register((c, p) => new DecodeTransform(p.Positional <IContext>(0))).Named <ITransform>("xmldecode");
builder.Register((c, p) => new XPathTransform(p.Positional <IContext>(0))).Named <ITransform>("xpath");
builder.Register((c, p) => new IIfTransform(p.Positional <IContext>(0))).Named <ITransform>("iif");
builder.Register((c, p) => new GeohashEncodeTransform(p.Positional <IContext>(0))).Named <ITransform>("geohashencode");
builder.Register((c, p) => new GeohashNeighborTransform(p.Positional <IContext>(0))).Named <ITransform>("geohashneighbor");
builder.Register((c, p) => new CommonPrefixTransform(p.Positional <IContext>(0))).Named <ITransform>("commonprefix");
builder.Register((c, p) => new CommonPrefixesTransform(p.Positional <IContext>(0))).Named <ITransform>("commonprefixes");
builder.Register((c, p) => new DistanceTransform(p.Positional <IContext>(0))).Named <ITransform>("distance");
builder.Register((c, p) => new FilterTransform(p.Positional <IContext>(0), FilterType.Include)).Named <ITransform>("include");
builder.Register((c, p) => new FilterTransform(p.Positional <IContext>(0), FilterType.Exclude)).Named <ITransform>("exclude");
// Humanizer
builder.Register((c, p) => new CamelizeTransform(p.Positional <IContext>(0))).Named <ITransform>("camelize");
builder.Register((c, p) => new FromMetricTransform(p.Positional <IContext>(0))).Named <ITransform>("frommetric");
builder.Register((c, p) => new FromRomanTransform(p.Positional <IContext>(0))).Named <ITransform>("fromroman");
builder.Register((c, p) => new HumanizeTransform(p.Positional <IContext>(0))).Named <ITransform>("humanize");
builder.Register((c, p) => new DehumanizeTransform(p.Positional <IContext>(0))).Named <ITransform>("dehumanize");
builder.Register((c, p) => new HyphenateTransform(p.Positional <IContext>(0))).Named <ITransform>("hyphenate");
builder.Register((c, p) => new HyphenateTransform(p.Positional <IContext>(0))).Named <ITransform>("dasherize");
builder.Register((c, p) => new OrdinalizeTransform(p.Positional <IContext>(0))).Named <ITransform>("ordinalize");
builder.Register((c, p) => new PascalizeTransform(p.Positional <IContext>(0))).Named <ITransform>("pascalize");
builder.Register((c, p) => new PluralizeTransform(p.Positional <IContext>(0))).Named <ITransform>("pluralize");
builder.Register((c, p) => new SingularizeTransform(p.Positional <IContext>(0))).Named <ITransform>("singularize");
builder.Register((c, p) => new TitleizeTransform(p.Positional <IContext>(0))).Named <ITransform>("titleize");
builder.Register((c, p) => new ToMetricTransform(p.Positional <IContext>(0))).Named <ITransform>("tometric");
builder.Register((c, p) => new ToOrdinalWordsTransform(p.Positional <IContext>(0))).Named <ITransform>("toordinalwords");
builder.Register((c, p) => new ToRomanTransform(p.Positional <IContext>(0))).Named <ITransform>("toroman");
builder.Register((c, p) => new ToWordsTransform(p.Positional <IContext>(0))).Named <ITransform>("towords");
builder.Register((c, p) => new UnderscoreTransform(p.Positional <IContext>(0))).Named <ITransform>("underscore");
builder.Register((c, p) => new BytesTransform(p.Positional <IContext>(0))).Named <ITransform>("bytes");
builder.Register((c, p) => new ByteSizeTransform(p.Positional <IContext>(0))).Named <ITransform>("bytesize");
builder.Register((c, p) => new DateAddTransform(p.Positional <IContext>(0))).Named <ITransform>("dateadd");
builder.Register((c, p) => new FromSplitTransform(p.Positional <IContext>(0))).Named <ITransform>("fromsplit");
builder.Register((c, p) => new FromLengthsTranform(p.Positional <IContext>(0))).Named <ITransform>("fromlengths");
// return true or false, validators
builder.Register((c, p) => new AnyTransform(p.Positional <IContext>(0))).Named <ITransform>("any");
builder.Register((c, p) => new StartsWithTransform(p.Positional <IContext>(0))).Named <ITransform>("startswith");
builder.Register((c, p) => new EndsWithTransform(p.Positional <IContext>(0))).Named <ITransform>("endswith");
builder.Register((c, p) => new InTransform(p.Positional <IContext>(0))).Named <ITransform>("in");
builder.Register((c, p) => new ContainsTransform(p.Positional <IContext>(0))).Named <ITransform>("contains");
builder.Register((c, p) => new IsTransform(p.Positional <IContext>(0))).Named <ITransform>("is");
builder.Register((c, p) => new EqualsTransform(p.Positional <IContext>(0))).Named <ITransform>("equal");
builder.Register((c, p) => new EqualsTransform(p.Positional <IContext>(0))).Named <ITransform>("equals");
builder.Register((c, p) => new IsEmptyTransform(p.Positional <IContext>(0))).Named <ITransform>("isempty");
builder.Register((c, p) => new IsDefaultTransform(p.Positional <IContext>(0))).Named <ITransform>("isdefault");
builder.Register((c, p) => new IsNumericTransform(p.Positional <IContext>(0))).Named <ITransform>("isnumeric");
builder.Register((c, p) => new RegexIsMatchTransform(p.Positional <IContext>(0))).Named <ITransform>("ismatch");
builder.Register((c, p) => new GeocodeTransform(p.Positional <IContext>(0))).Named <ITransform>("fromaddress");
builder.Register((c, p) => new DateMathTransform(p.Positional <IContext>(0))).Named <ITransform>("datemath");
builder.Register((c, p) => new IsDaylightSavingsTransform(p.Positional <IContext>(0))).Named <ITransform>("isdaylightsavings");
builder.Register((c, p) => new SlugifyTransform(p.Positional <IContext>(0))).Named <ITransform>("slugify");
/* VIN, Vehicle Identification Number, note: you get red intellisense here because vin library is portable */
builder.Register((c, p) => new VinValidateTransform(p.Positional <IContext>(0))).Named <ITransform>("isvin");
builder.Register((c, p) => new VinValidateTransform(p.Positional <IContext>(0))).Named <ITransform>("vinisvalid");
builder.Register((c, p) => new VinGetWorldManufacturerTransform(p.Positional <IContext>(0))).Named <ITransform>("vingetworldmanufacturer");
builder.Register((c, p) => new VinGetModelYearTransform(p.Positional <IContext>(0))).Named <ITransform>("vingetmodelyear");
// wip
builder.Register((c, p) => new WebTransform(p.Positional <IContext>(0))).Named <ITransform>("web");
builder.Register((c, p) => new UrlEncodeTransform(p.Positional <IContext>(0))).Named <ITransform>("urlencode");
builder.Register((c, p) => new FromJsonTransform(p.Positional <IContext>(0), o => JsonConvert.SerializeObject(o, Formatting.None))).Named <ITransform>("fromjson");
builder.Register((c, p) => new LamdaParserEvalTransform(p.Positional <IContext>(0))).Named <ITransform>("eval");
builder.Register((c, p) => new DistinctTransform(p.Positional <IContext>(0))).Named <ITransform>("distinct");
builder.Register((c, p) => new RegexMatchCountTransform(p.Positional <IContext>(0))).Named <ITransform>("matchcount");
builder.Register((c, p) => new AppendTransform(p.Positional <IContext>(0))).Named <ITransform>("append");
builder.Register((c, p) => new PrependTransform(p.Positional <IContext>(0))).Named <ITransform>("prepend");
builder.Register((c, p) => {
var context = p.Positional <IContext>(0);
return(context.Operation.XmlMode == "all" || context.Field.Engine != "auto" ?
new Transforms.Xml.FromXmlTransform(context, c.ResolveNamed <IRowFactory>(context.Entity.Key, new NamedParameter("capacity", context.GetAllEntityFields().Count()))) :
new Transforms.FromXmlTransform(context) as ITransform);
}).Named <ITransform>("fromxml");
builder.Register <ITransform>((c, p) => {
var context = p.Positional <IContext>(0);
if (c.ResolveNamed <IHost>("cs").Start())
{
return(new CsharpTransform(context));
}
context.Error("Unable to register csharp transform");
return(new NullTransform(context));
}).Named <ITransform>("cs");
builder.Register((c, p) => c.ResolveNamed <ITransform>("cs", p)).Named <ITransform>("csharp");
builder.Register((c, p) => {
var context = p.Positional <IContext>(0);
var strict = new HashSet <string>(new[] { "int", "int32", "string", "bool", "boolean", "double" });
switch (context.Field.Engine)
{
case "auto":
var fields = context.Entity.GetFieldMatches(context.Operation.Script);
return(fields.All(f => strict.Contains(f.Type)) ?
(ITransform) new JavascriptTransform(new ChakraCoreJsEngineFactory(), context, c.Resolve <IReader>()) :
new JintTransform(context, c.Resolve <IReader>()));
case "jint":
return(new JintTransform(context, c.Resolve <IReader>()));
default:
return(new JavascriptTransform(new ChakraCoreJsEngineFactory(), context, c.Resolve <IReader>()));
}
}).Named <ITransform>("js");
builder.Register((c, p) => c.ResolveNamed <ITransform>("js", p)).Named <ITransform>("javascript");
}
