public void StoresPersistentKeyValueStoreTest() { var builder = new StreamBuilder(); // Same like that : // builder.Table("table-topic", RocksDb<string, string>.As("table-topic-store")); builder.Table("table-topic", Materialized <string, string, IKeyValueStore <Bytes, byte[]> > .Create( Streamiz.Kafka.Net.State.Stores.PersistentKeyValueStore("table-topic-store"))); var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test-map"; config.UseRandomRocksDbConfigForTest(); Topology t = builder.Build(); using (var driver = new TopologyTestDriver(t, config)) { var inputTopic = driver.CreateInputTopic <string, string>("table-topic"); inputTopic.PipeInput("key1", "1"); inputTopic.PipeInput("key2", "2"); var store = driver.GetKeyValueStore <string, string>("table-topic-store"); Assert.IsNotNull(store); var resultK1 = store.Get("key1"); var resultK2 = store.Get("key2"); Assert.AreEqual("1", resultK1); Assert.AreEqual("2", resultK2); } config.RemoveRocksDbFolderForTest(); }
public void Begin() { config = new StreamConfig(); config.ApplicationId = $"unit-test-rocksdb-w"; config.UseRandomRocksDbConfigForTest(); id = new TaskId { Id = 0, Partition = 0 }; partition = new TopicPartition("source", 0); stateManager = new ProcessorStateManager( id, new List <TopicPartition> { partition }, null, new MockChangelogRegister(), new MockOffsetCheckpointManager()); task = new Mock <AbstractTask>(); task.Setup(k => k.Id).Returns(id); context = new ProcessorContext(task.Object, config, stateManager, new StreamMetricsRegistry()); store = new RocksDbWindowStore( new RocksDbSegmentedBytesStore("test-w-store", (long)defaultRetention.TotalMilliseconds, 5000, new RocksDbWindowKeySchema()), (long)defaultSize.TotalMilliseconds); store.Init(context, store); }
public void KTableSource() { var builder = new StreamBuilder(); builder.Table("table-topic", RocksDb <string, string> .As <StringSerDes, StringSerDes>("table-topic-store")); var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test-map"; config.UseRandomRocksDbConfigForTest(); Topology t = builder.Build(); using (var driver = new TopologyTestDriver(t, config)) { var inputTopic = driver.CreateInputTopic <string, string>("table-topic"); inputTopic.PipeInput("key1", "1"); inputTopic.PipeInput("key2", "2"); var store = driver.GetKeyValueStore <string, string>("table-topic-store"); Assert.IsNotNull(store); var resultK1 = store.Get("key1"); var resultK2 = store.Get("key2"); Assert.AreEqual("1", resultK1); Assert.AreEqual("2", resultK2); } config.RemoveRocksDbFolderForTest(); }
public void Begin() { config = new StreamConfig(); config.ApplicationId = $"unit-test-rocksdb-kv"; config.UseRandomRocksDbConfigForTest(); id = new TaskId { Id = 0, Partition = 0 }; partition = new TopicPartition("source", 0); stateManager = new ProcessorStateManager( id, new List <TopicPartition> { partition }, null, new MockChangelogRegister(), new MockOffsetCheckpointManager()); task = new Mock <AbstractTask>(); task.Setup(k => k.Id).Returns(id); context = new ProcessorContext(task.Object, config, stateManager, new StreamMetricsRegistry()); store = new RocksDbKeyValueStore("test-store"); store.Init(context, store); }
public void Windowing() { var builder = new StreamBuilder(); builder.Stream <string, string>("topic") .GroupByKey() .WindowedBy(TumblingWindowOptions.Of(1000)) .Count(RocksDbWindows <string, long> .As <StringSerDes, Int64SerDes>("rocksdb-w-store")); var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test-rocksdb-window-store"; config.UseRandomRocksDbConfigForTest(); Topology t = builder.Build(); using (var driver = new TopologyTestDriver(t, config)) { DateTime dt = DateTime.Now; var inputTopic = driver.CreateInputTopic <string, string>("topic"); inputTopic.PipeInput("key1", "1", dt); inputTopic.PipeInput("key2", "2", dt); var store = driver.GetWindowStore <string, long>("rocksdb-w-store"); Assert.IsNotNull(store); var k1 = store.FetchAll(dt.AddMinutes(-10), dt.AddMinutes(10)).ToList(); Assert.AreEqual(2, k1.Count); Assert.AreEqual(1L, k1[0].Value); Assert.AreEqual(1L, k1[1].Value); } config.RemoveRocksDbFolderForTest(); }
public void GlobalTableWithStoreQueryableWithRocksDb() { var builder = new StreamBuilder(); var table = builder.GlobalTable("topic", RocksDb <string, string> .As("global-store")); var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test-globaltable"; config.UseRandomRocksDbConfigForTest(); Topology t = builder.Build(); using (var driver = new TopologyTestDriver(t, config)) { var inputTopic = driver.CreateInputTopic <string, string>("topic"); inputTopic.PipeInput("test", "coucou"); var store = driver.GetKeyValueStore <string, string>("global-store"); var ele = store.Get("test"); Assert.IsNotNull(ele); Assert.AreEqual("coucou", ele); } config.RemoveRocksDbFolderForTest(); }
public void StoresPersistentKeyValueStoreTest() { var builder = new StreamBuilder(); builder.Stream <string, string>("topic") .GroupByKey() .WindowedBy(TumblingWindowOptions.Of(1000)) .Count( Materialized <string, long, IWindowStore <Bytes, byte[]> > .Create( Streamiz.Kafka.Net.State.Stores.PersistentWindowStore( "rocksdb-w-store", TimeSpan.FromDays(1), TimeSpan.FromSeconds(1)))); var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test-rocksdb-window-store"; config.UseRandomRocksDbConfigForTest(); Topology t = builder.Build(); using (var driver = new TopologyTestDriver(t, config)) { DateTime dt = DateTime.Now; var inputTopic = driver.CreateInputTopic <string, string>("topic"); inputTopic.PipeInput("abc", "1", dt); inputTopic.PipeInput("key1", "1", dt); inputTopic.PipeInput("test", "1", dt); var store = driver.GetWindowStore <string, long>("rocksdb-w-store"); Assert.IsNotNull(store); var k1 = store.FetchAll(dt.AddMinutes(-10), dt.AddMinutes(10)).ToList(); Assert.AreEqual(3, k1.Count); } config.RemoveRocksDbFolderForTest(); }
public void Begin() { Random rd = new Random(); config = new StreamConfig(); config.ApplicationId = $"RocksDbOptionsTests"; config.UseRandomRocksDbConfigForTest(); config.RocksDbConfigHandler = (name, options) => { options .EnableStatistics() .IncreaseParallelism(parallelism) .OptimizeForPointLookup(blockCacheSizeMb) .OptimizeLevelStyleCompaction(memTableMemoryBudget) .OptimizeUniversalStyleCompaction(memtableMemoryBudget2) .PrepareForBulkLoad() .SetAccessHintOnCompactionStart(accessHintOnCompactionStart) .SetAdviseRandomOnOpen(adviseRandomOnOpen) .SetAllowConcurrentMemtableWrite(allowConcurrentMemtableWrite) .SetAllowMmapReads(false) .SetAllowMmapWrites(false) .SetArenaBlockSize(arenaBlockSize) .SetBaseBackgroundCompactions(baseBackgroundCompactions) .SetBloomLocality(bloomLocality) .SetBytesPerSync(bytesPerSync) // .SetCompactionFilter(IntPtr.Zero) // .SetCompactionFilterFactory(IntPtr.Zero) .SetCompactionReadaheadSize(1200) .SetCompactionStyle(RocksDbSharp.Compaction.Level) //.SetComparator(IntPtr.Zero) .SetCompression(RocksDbSharp.Compression.Lz4) .SetCompressionOptions(1, 2, 3, 4) .SetCompressionPerLevel(new[] { RocksDbSharp.Compression.Lz4 }, 1) .SetCreateIfMissing() .SetCreateMissingColumnFamilies() .SetDbLogDir("test") .SetDbWriteBufferSize(1000) .SetDeleteObsoleteFilesPeriodMicros(50) .SetDisableAutoCompactions(1) .SetEnableWriteThreadAdaptiveYield(true) //.SetEnv(IntPtr.Zero) .SetErrorIfExists() //.SetFifoCompactionOptions(IntPtr.Zero) .SetHardPendingCompactionBytesLimit(1) .SetHardRateLimit(40597) .SetHashLinkListRep(12) .SetHashSkipListRep(56, 4, 2) //.SetInfoLog(IntPtr.Zero) .SetInfoLogLevel(RocksLogLevel.NUM_INFO_LOG) .SetInplaceUpdateNumLocks(134) .SetIsFdCloseOnExec(false) .SetKeepLogFileNum(1) .SetLevel0FileNumCompactionTrigger(14) .SetLevel0SlowdownWritesTrigger(144) .SetInplaceUpdateSupport(true) .SetLevel0StopWritesTrigger(24) .SetLevelCompactionDynamicLevelBytes(true) .SetLogFileTimeToRoll(154) .SetManifestPreallocationSize(153) .SetMaxBackgroundFlushes(3) .SetMaxBytesForLevelBase(1453) .SetMaxBytesForLevelMultiplier(2) .SetMaxBytesForLevelMultiplierAdditional(new[] { 1 }, 1) .SetMaxCompactionBytes(345678) .SetMaxFileOpeningThreads(2) .SetMaxLogFileSize(1) .SetMaxManifestFileSize(131) .SetMaxMemCompactionLevel(12) .SetMaxOpenFiles(20) .SetMaxSequentialSkipInIterations(13) .SetMaxSuccessiveMerges(12) .SetMaxTotalWalSize(12) .SetMaxWriteBufferNumber(15543) .SetMaxWriteBufferNumberToMaintain(126) .SetMemtableHugePageSize(64317) .SetMemtableHugePageSize(64317) .SetMemtablePrefixBloomSizeRatio(12) //.SetMergeOperator(IntPtr.Zero) .SetMinLevelToCompress(1) .SetMemtableVectorRep() .SetMinWriteBufferNumberToMerge(1312) .SetNumLevels(1) .SetOptimizeFiltersForHits(56) .SetParanoidChecks() .SetSkipLogErrorOnRecovery(true) .SetSoftPendingCompactionBytesLimit(134) .SetSoftRateLimit(131) .SetStatsDumpPeriodSec(532) .SetTableCacheNumShardbits(12) .SetTableCacheRemoveScanCountLimit(66) .SetTargetFileSizeBase(6) .SetTargetFileSizeMultiplier(2) .SetUint64addMergeOperator() //.SetUniversalCompactionOptions(IntPtr.Zero) .SetUseAdaptiveMutex(false) .SetUseDirectIoForFlushAndCompaction(true) .SetUseDirectReads(true) .SetUseFsync(1) .SetWalRecoveryMode(RocksDbSharp.Recovery.SkipAnyCorruptedRecords) .SetWALSizeLimitMB(40) .SetWALTtlSeconds(1454151413) .SetWriteBufferSize(45678976543) .SetPrefixExtractor(SliceTransform.CreateNoOp()) .SetPurgeRedundantKvsWhileFlush(false) .SetRateLimitDelayMaxMilliseconds(1762) .SetRecycleLogFileNum(1) .SetReportBgIoStats(true) .SetPlainTableFactory(1, 23, 4, 2) .SetMaxBackgroundCompactions(1); }; id = new TaskId { Id = 0, Partition = 0 }; partition = new TopicPartition("source", 0); stateManager = new ProcessorStateManager( id, new List <TopicPartition> { partition }, null, new MockChangelogRegister(), new MockOffsetCheckpointManager()); task = new Mock <AbstractTask>(); task.Setup(k => k.Id).Returns(id); context = new ProcessorContext(task.Object, config, stateManager, new StreamMetricsRegistry()); store = new RocksDbKeyValueStore("test-store"); store.Init(context, store); }