public void Begin() { config = new StreamConfig(); config.ApplicationId = $"unit-test-rocksdb-kv"; config.UseRandomRocksDbConfigForTest(); id = new TaskId { Id = 0, Partition = 0 }; partition = new TopicPartition("source", 0); stateManager = new ProcessorStateManager( id, new List <TopicPartition> { partition }, null, new MockChangelogRegister(), new MockOffsetCheckpointManager()); task = new Mock <AbstractTask>(); task.Setup(k => k.Id).Returns(id); context = new ProcessorContext(task.Object, config, stateManager, new StreamMetricsRegistry()); store = new RocksDbKeyValueStore("test-store"); store.Init(context, store); }
public void Init() { stateDir = Path.Combine(".", Guid.NewGuid().ToString()); config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test-storechangelog-app"; config.StateDir = stateDir; config.PollMs = 100; supplier = new SyncKafkaSupplier(); var producer = supplier.GetProducer(config.ToProducerConfig()); restoreConsumer = supplier.GetRestoreConsumer(config.ToConsumerConfig()); var topicPart = new TopicPartition("topic", 0); changelogTopic = "store-changelog-topic"; var changelogsTopics = new Dictionary <string, string>(); changelogsTopics.Add("store", changelogTopic); var id = new TaskId { Id = 0, Partition = 0 }; store = new RocksDbKeyValueStore("store"); storeChangelogReader = new StoreChangelogReader(config, restoreConsumer, "thread-0", new StreamMetricsRegistry()); stateMgr = new ProcessorStateManager( id, topicPart.ToSingle(), changelogsTopics, storeChangelogReader, new OffsetCheckpointFile(Path.Combine(config.StateDir, config.ApplicationId, $"{id.Id}-{id.Partition}")) ); Mock <AbstractTask> moq = new Mock <AbstractTask>(); moq.Setup(t => t.Id).Returns(new TaskId { Id = 0, Partition = 0 }); context = new ProcessorContext(moq.Object, config, stateMgr, new StreamMetricsRegistry()); store.Init(context, store); producer.Produce(changelogTopic, CreateMessage(changelogTopic, "key1", "value1")); producer.Produce(changelogTopic, CreateMessage(changelogTopic, "key2", "value2")); }
public void Begin() { Random rd = new Random(); config = new StreamConfig(); config.ApplicationId = $"RocksDbOptionsTests"; config.UseRandomRocksDbConfigForTest(); config.RocksDbConfigHandler = (name, options) => { options .EnableStatistics() .IncreaseParallelism(parallelism) .OptimizeForPointLookup(blockCacheSizeMb) .OptimizeLevelStyleCompaction(memTableMemoryBudget) .OptimizeUniversalStyleCompaction(memtableMemoryBudget2) .PrepareForBulkLoad() .SetAccessHintOnCompactionStart(accessHintOnCompactionStart) .SetAdviseRandomOnOpen(adviseRandomOnOpen) .SetAllowConcurrentMemtableWrite(allowConcurrentMemtableWrite) .SetAllowMmapReads(false) .SetAllowMmapWrites(false) .SetArenaBlockSize(arenaBlockSize) .SetBaseBackgroundCompactions(baseBackgroundCompactions) .SetBloomLocality(bloomLocality) .SetBytesPerSync(bytesPerSync) // .SetCompactionFilter(IntPtr.Zero) // .SetCompactionFilterFactory(IntPtr.Zero) .SetCompactionReadaheadSize(1200) .SetCompactionStyle(RocksDbSharp.Compaction.Level) //.SetComparator(IntPtr.Zero) .SetCompression(RocksDbSharp.Compression.Lz4) .SetCompressionOptions(1, 2, 3, 4) .SetCompressionPerLevel(new[] { RocksDbSharp.Compression.Lz4 }, 1) .SetCreateIfMissing() .SetCreateMissingColumnFamilies() .SetDbLogDir("test") .SetDbWriteBufferSize(1000) .SetDeleteObsoleteFilesPeriodMicros(50) .SetDisableAutoCompactions(1) .SetEnableWriteThreadAdaptiveYield(true) //.SetEnv(IntPtr.Zero) .SetErrorIfExists() //.SetFifoCompactionOptions(IntPtr.Zero) .SetHardPendingCompactionBytesLimit(1) .SetHardRateLimit(40597) .SetHashLinkListRep(12) .SetHashSkipListRep(56, 4, 2) //.SetInfoLog(IntPtr.Zero) .SetInfoLogLevel(RocksLogLevel.NUM_INFO_LOG) .SetInplaceUpdateNumLocks(134) .SetIsFdCloseOnExec(false) .SetKeepLogFileNum(1) .SetLevel0FileNumCompactionTrigger(14) .SetLevel0SlowdownWritesTrigger(144) .SetInplaceUpdateSupport(true) .SetLevel0StopWritesTrigger(24) .SetLevelCompactionDynamicLevelBytes(true) .SetLogFileTimeToRoll(154) .SetManifestPreallocationSize(153) .SetMaxBackgroundFlushes(3) .SetMaxBytesForLevelBase(1453) .SetMaxBytesForLevelMultiplier(2) .SetMaxBytesForLevelMultiplierAdditional(new[] { 1 }, 1) .SetMaxCompactionBytes(345678) .SetMaxFileOpeningThreads(2) .SetMaxLogFileSize(1) .SetMaxManifestFileSize(131) .SetMaxMemCompactionLevel(12) .SetMaxOpenFiles(20) .SetMaxSequentialSkipInIterations(13) .SetMaxSuccessiveMerges(12) .SetMaxTotalWalSize(12) .SetMaxWriteBufferNumber(15543) .SetMaxWriteBufferNumberToMaintain(126) .SetMemtableHugePageSize(64317) .SetMemtableHugePageSize(64317) .SetMemtablePrefixBloomSizeRatio(12) //.SetMergeOperator(IntPtr.Zero) .SetMinLevelToCompress(1) .SetMemtableVectorRep() .SetMinWriteBufferNumberToMerge(1312) .SetNumLevels(1) .SetOptimizeFiltersForHits(56) .SetParanoidChecks() .SetSkipLogErrorOnRecovery(true) .SetSoftPendingCompactionBytesLimit(134) .SetSoftRateLimit(131) .SetStatsDumpPeriodSec(532) .SetTableCacheNumShardbits(12) .SetTableCacheRemoveScanCountLimit(66) .SetTargetFileSizeBase(6) .SetTargetFileSizeMultiplier(2) .SetUint64addMergeOperator() //.SetUniversalCompactionOptions(IntPtr.Zero) .SetUseAdaptiveMutex(false) .SetUseDirectIoForFlushAndCompaction(true) .SetUseDirectReads(true) .SetUseFsync(1) .SetWalRecoveryMode(RocksDbSharp.Recovery.SkipAnyCorruptedRecords) .SetWALSizeLimitMB(40) .SetWALTtlSeconds(1454151413) .SetWriteBufferSize(45678976543) .SetPrefixExtractor(SliceTransform.CreateNoOp()) .SetPurgeRedundantKvsWhileFlush(false) .SetRateLimitDelayMaxMilliseconds(1762) .SetRecycleLogFileNum(1) .SetReportBgIoStats(true) .SetPlainTableFactory(1, 23, 4, 2) .SetMaxBackgroundCompactions(1); }; id = new TaskId { Id = 0, Partition = 0 }; partition = new TopicPartition("source", 0); stateManager = new ProcessorStateManager( id, new List <TopicPartition> { partition }, null, new MockChangelogRegister(), new MockOffsetCheckpointManager()); task = new Mock <AbstractTask>(); task.Setup(k => k.Id).Returns(id); context = new ProcessorContext(task.Object, config, stateManager, new StreamMetricsRegistry()); store = new RocksDbKeyValueStore("test-store"); store.Init(context, store); }