public void Begin()
{
config = new StreamConfig();
config.ApplicationId = $"unit-test-rocksdb-kv";
config.UseRandomRocksDbConfigForTest();
id = new TaskId {
Id = 0, Partition = 0
};
partition = new TopicPartition("source", 0);
stateManager = new ProcessorStateManager(
id,
new List <TopicPartition> {
partition
},
null,
new MockChangelogRegister(),
new MockOffsetCheckpointManager());
task = new Mock <AbstractTask>();
task.Setup(k => k.Id).Returns(id);
context = new ProcessorContext(task.Object, config, stateManager, new StreamMetricsRegistry());
store = new RocksDbKeyValueStore("test-store");
store.Init(context, store);
}
public void Init()
{
stateDir = Path.Combine(".", Guid.NewGuid().ToString());
config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-storechangelog-app";
config.StateDir = stateDir;
config.PollMs = 100;
supplier = new SyncKafkaSupplier();
var producer = supplier.GetProducer(config.ToProducerConfig());
restoreConsumer = supplier.GetRestoreConsumer(config.ToConsumerConfig());
var topicPart = new TopicPartition("topic", 0);
changelogTopic = "store-changelog-topic";
var changelogsTopics = new Dictionary <string, string>();
changelogsTopics.Add("store", changelogTopic);
var id = new TaskId
{
Id = 0,
Partition = 0
};
store = new RocksDbKeyValueStore("store");
storeChangelogReader =
new StoreChangelogReader(config, restoreConsumer, "thread-0", new StreamMetricsRegistry());
stateMgr = new ProcessorStateManager(
id,
topicPart.ToSingle(),
changelogsTopics,
storeChangelogReader,
new OffsetCheckpointFile(Path.Combine(config.StateDir, config.ApplicationId, $"{id.Id}-{id.Partition}"))
);
Mock <AbstractTask> moq = new Mock <AbstractTask>();
moq.Setup(t => t.Id).Returns(new TaskId {
Id = 0, Partition = 0
});
context = new ProcessorContext(moq.Object, config, stateMgr, new StreamMetricsRegistry());
store.Init(context, store);
producer.Produce(changelogTopic, CreateMessage(changelogTopic, "key1", "value1"));
producer.Produce(changelogTopic, CreateMessage(changelogTopic, "key2", "value2"));
}
public void Begin()
{
Random rd = new Random();
config = new StreamConfig();
config.ApplicationId = $"RocksDbOptionsTests";
config.UseRandomRocksDbConfigForTest();
config.RocksDbConfigHandler = (name, options) =>
{
options
.EnableStatistics()
.IncreaseParallelism(parallelism)
.OptimizeForPointLookup(blockCacheSizeMb)
.OptimizeLevelStyleCompaction(memTableMemoryBudget)
.OptimizeUniversalStyleCompaction(memtableMemoryBudget2)
.PrepareForBulkLoad()
.SetAccessHintOnCompactionStart(accessHintOnCompactionStart)
.SetAdviseRandomOnOpen(adviseRandomOnOpen)
.SetAllowConcurrentMemtableWrite(allowConcurrentMemtableWrite)
.SetAllowMmapReads(false)
.SetAllowMmapWrites(false)
.SetArenaBlockSize(arenaBlockSize)
.SetBaseBackgroundCompactions(baseBackgroundCompactions)
.SetBloomLocality(bloomLocality)
.SetBytesPerSync(bytesPerSync)
// .SetCompactionFilter(IntPtr.Zero)
// .SetCompactionFilterFactory(IntPtr.Zero)
.SetCompactionReadaheadSize(1200)
.SetCompactionStyle(RocksDbSharp.Compaction.Level)
//.SetComparator(IntPtr.Zero)
.SetCompression(RocksDbSharp.Compression.Lz4)
.SetCompressionOptions(1, 2, 3, 4)
.SetCompressionPerLevel(new[] { RocksDbSharp.Compression.Lz4 }, 1)
.SetCreateIfMissing()
.SetCreateMissingColumnFamilies()
.SetDbLogDir("test")
.SetDbWriteBufferSize(1000)
.SetDeleteObsoleteFilesPeriodMicros(50)
.SetDisableAutoCompactions(1)
.SetEnableWriteThreadAdaptiveYield(true)
//.SetEnv(IntPtr.Zero)
.SetErrorIfExists()
//.SetFifoCompactionOptions(IntPtr.Zero)
.SetHardPendingCompactionBytesLimit(1)
.SetHardRateLimit(40597)
.SetHashLinkListRep(12)
.SetHashSkipListRep(56, 4, 2)
//.SetInfoLog(IntPtr.Zero)
.SetInfoLogLevel(RocksLogLevel.NUM_INFO_LOG)
.SetInplaceUpdateNumLocks(134)
.SetIsFdCloseOnExec(false)
.SetKeepLogFileNum(1)
.SetLevel0FileNumCompactionTrigger(14)
.SetLevel0SlowdownWritesTrigger(144)
.SetInplaceUpdateSupport(true)
.SetLevel0StopWritesTrigger(24)
.SetLevelCompactionDynamicLevelBytes(true)
.SetLogFileTimeToRoll(154)
.SetManifestPreallocationSize(153)
.SetMaxBackgroundFlushes(3)
.SetMaxBytesForLevelBase(1453)
.SetMaxBytesForLevelMultiplier(2)
.SetMaxBytesForLevelMultiplierAdditional(new[] { 1 }, 1)
.SetMaxCompactionBytes(345678)
.SetMaxFileOpeningThreads(2)
.SetMaxLogFileSize(1)
.SetMaxManifestFileSize(131)
.SetMaxMemCompactionLevel(12)
.SetMaxOpenFiles(20)
.SetMaxSequentialSkipInIterations(13)
.SetMaxSuccessiveMerges(12)
.SetMaxTotalWalSize(12)
.SetMaxWriteBufferNumber(15543)
.SetMaxWriteBufferNumberToMaintain(126)
.SetMemtableHugePageSize(64317)
.SetMemtableHugePageSize(64317)
.SetMemtablePrefixBloomSizeRatio(12)
//.SetMergeOperator(IntPtr.Zero)
.SetMinLevelToCompress(1)
.SetMemtableVectorRep()
.SetMinWriteBufferNumberToMerge(1312)
.SetNumLevels(1)
.SetOptimizeFiltersForHits(56)
.SetParanoidChecks()
.SetSkipLogErrorOnRecovery(true)
.SetSoftPendingCompactionBytesLimit(134)
.SetSoftRateLimit(131)
.SetStatsDumpPeriodSec(532)
.SetTableCacheNumShardbits(12)
.SetTableCacheRemoveScanCountLimit(66)
.SetTargetFileSizeBase(6)
.SetTargetFileSizeMultiplier(2)
.SetUint64addMergeOperator()
//.SetUniversalCompactionOptions(IntPtr.Zero)
.SetUseAdaptiveMutex(false)
.SetUseDirectIoForFlushAndCompaction(true)
.SetUseDirectReads(true)
.SetUseFsync(1)
.SetWalRecoveryMode(RocksDbSharp.Recovery.SkipAnyCorruptedRecords)
.SetWALSizeLimitMB(40)
.SetWALTtlSeconds(1454151413)
.SetWriteBufferSize(45678976543)
.SetPrefixExtractor(SliceTransform.CreateNoOp())
.SetPurgeRedundantKvsWhileFlush(false)
.SetRateLimitDelayMaxMilliseconds(1762)
.SetRecycleLogFileNum(1)
.SetReportBgIoStats(true)
.SetPlainTableFactory(1, 23, 4, 2)
.SetMaxBackgroundCompactions(1);
};
id = new TaskId {
Id = 0, Partition = 0
};
partition = new TopicPartition("source", 0);
stateManager = new ProcessorStateManager(
id,
new List <TopicPartition> {
partition
},
null,
new MockChangelogRegister(),
new MockOffsetCheckpointManager());
task = new Mock <AbstractTask>();
task.Setup(k => k.Id).Returns(id);
context = new ProcessorContext(task.Object, config, stateManager, new StreamMetricsRegistry());
store = new RocksDbKeyValueStore("test-store");
store.Init(context, store);
}