public void StoresPersistentKeyValueStoreTest()
{
var builder = new StreamBuilder();
// Same like that :
// builder.Table("table-topic", RocksDb<string, string>.As("table-topic-store"));
builder.Table("table-topic",
Materialized <string, string, IKeyValueStore <Bytes, byte[]> > .Create(
Streamiz.Kafka.Net.State.Stores.PersistentKeyValueStore("table-topic-store")));
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-map";
config.UseRandomRocksDbConfigForTest();
Topology t = builder.Build();
using (var driver = new TopologyTestDriver(t, config))
{
var inputTopic = driver.CreateInputTopic <string, string>("table-topic");
inputTopic.PipeInput("key1", "1");
inputTopic.PipeInput("key2", "2");
var store = driver.GetKeyValueStore <string, string>("table-topic-store");
Assert.IsNotNull(store);
var resultK1 = store.Get("key1");
var resultK2 = store.Get("key2");
Assert.AreEqual("1", resultK1);
Assert.AreEqual("2", resultK2);
}
config.RemoveRocksDbFolderForTest();
}
public void Begin()
{
config = new StreamConfig();
config.ApplicationId = $"unit-test-rocksdb-w";
config.UseRandomRocksDbConfigForTest();
id = new TaskId {
Id = 0, Partition = 0
};
partition = new TopicPartition("source", 0);
stateManager = new ProcessorStateManager(
id,
new List <TopicPartition> {
partition
},
null,
new MockChangelogRegister(),
new MockOffsetCheckpointManager());
task = new Mock <AbstractTask>();
task.Setup(k => k.Id).Returns(id);
context = new ProcessorContext(task.Object, config, stateManager, new StreamMetricsRegistry());
store = new RocksDbWindowStore(
new RocksDbSegmentedBytesStore("test-w-store", (long)defaultRetention.TotalMilliseconds, 5000, new RocksDbWindowKeySchema()),
(long)defaultSize.TotalMilliseconds);
store.Init(context, store);
}
public void KTableSource()
{
var builder = new StreamBuilder();
builder.Table("table-topic", RocksDb <string, string> .As <StringSerDes, StringSerDes>("table-topic-store"));
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-map";
config.UseRandomRocksDbConfigForTest();
Topology t = builder.Build();
using (var driver = new TopologyTestDriver(t, config))
{
var inputTopic = driver.CreateInputTopic <string, string>("table-topic");
inputTopic.PipeInput("key1", "1");
inputTopic.PipeInput("key2", "2");
var store = driver.GetKeyValueStore <string, string>("table-topic-store");
Assert.IsNotNull(store);
var resultK1 = store.Get("key1");
var resultK2 = store.Get("key2");
Assert.AreEqual("1", resultK1);
Assert.AreEqual("2", resultK2);
}
config.RemoveRocksDbFolderForTest();
}
public void Begin()
{
config = new StreamConfig();
config.ApplicationId = $"unit-test-rocksdb-kv";
config.UseRandomRocksDbConfigForTest();
id = new TaskId {
Id = 0, Partition = 0
};
partition = new TopicPartition("source", 0);
stateManager = new ProcessorStateManager(
id,
new List <TopicPartition> {
partition
},
null,
new MockChangelogRegister(),
new MockOffsetCheckpointManager());
task = new Mock <AbstractTask>();
task.Setup(k => k.Id).Returns(id);
context = new ProcessorContext(task.Object, config, stateManager, new StreamMetricsRegistry());
store = new RocksDbKeyValueStore("test-store");
store.Init(context, store);
}
public void Windowing()
{
var builder = new StreamBuilder();
builder.Stream <string, string>("topic")
.GroupByKey()
.WindowedBy(TumblingWindowOptions.Of(1000))
.Count(RocksDbWindows <string, long> .As <StringSerDes, Int64SerDes>("rocksdb-w-store"));
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-rocksdb-window-store";
config.UseRandomRocksDbConfigForTest();
Topology t = builder.Build();
using (var driver = new TopologyTestDriver(t, config))
{
DateTime dt = DateTime.Now;
var inputTopic = driver.CreateInputTopic <string, string>("topic");
inputTopic.PipeInput("key1", "1", dt);
inputTopic.PipeInput("key2", "2", dt);
var store = driver.GetWindowStore <string, long>("rocksdb-w-store");
Assert.IsNotNull(store);
var k1 = store.FetchAll(dt.AddMinutes(-10), dt.AddMinutes(10)).ToList();
Assert.AreEqual(2, k1.Count);
Assert.AreEqual(1L, k1[0].Value);
Assert.AreEqual(1L, k1[1].Value);
}
config.RemoveRocksDbFolderForTest();
}
public void GlobalTableWithStoreQueryableWithRocksDb()
{
var builder = new StreamBuilder();
var table = builder.GlobalTable("topic", RocksDb <string, string> .As("global-store"));
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-globaltable";
config.UseRandomRocksDbConfigForTest();
Topology t = builder.Build();
using (var driver = new TopologyTestDriver(t, config))
{
var inputTopic = driver.CreateInputTopic <string, string>("topic");
inputTopic.PipeInput("test", "coucou");
var store = driver.GetKeyValueStore <string, string>("global-store");
var ele = store.Get("test");
Assert.IsNotNull(ele);
Assert.AreEqual("coucou", ele);
}
config.RemoveRocksDbFolderForTest();
}
public void StoresPersistentKeyValueStoreTest()
{
var builder = new StreamBuilder();
builder.Stream <string, string>("topic")
.GroupByKey()
.WindowedBy(TumblingWindowOptions.Of(1000))
.Count(
Materialized <string, long, IWindowStore <Bytes, byte[]> > .Create(
Streamiz.Kafka.Net.State.Stores.PersistentWindowStore(
"rocksdb-w-store",
TimeSpan.FromDays(1),
TimeSpan.FromSeconds(1))));
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-rocksdb-window-store";
config.UseRandomRocksDbConfigForTest();
Topology t = builder.Build();
using (var driver = new TopologyTestDriver(t, config))
{
DateTime dt = DateTime.Now;
var inputTopic = driver.CreateInputTopic <string, string>("topic");
inputTopic.PipeInput("abc", "1", dt);
inputTopic.PipeInput("key1", "1", dt);
inputTopic.PipeInput("test", "1", dt);
var store = driver.GetWindowStore <string, long>("rocksdb-w-store");
Assert.IsNotNull(store);
var k1 = store.FetchAll(dt.AddMinutes(-10), dt.AddMinutes(10)).ToList();
Assert.AreEqual(3, k1.Count);
}
config.RemoveRocksDbFolderForTest();
}
public void Begin()
{
Random rd = new Random();
config = new StreamConfig();
config.ApplicationId = $"RocksDbOptionsTests";
config.UseRandomRocksDbConfigForTest();
config.RocksDbConfigHandler = (name, options) =>
{
options
.EnableStatistics()
.IncreaseParallelism(parallelism)
.OptimizeForPointLookup(blockCacheSizeMb)
.OptimizeLevelStyleCompaction(memTableMemoryBudget)
.OptimizeUniversalStyleCompaction(memtableMemoryBudget2)
.PrepareForBulkLoad()
.SetAccessHintOnCompactionStart(accessHintOnCompactionStart)
.SetAdviseRandomOnOpen(adviseRandomOnOpen)
.SetAllowConcurrentMemtableWrite(allowConcurrentMemtableWrite)
.SetAllowMmapReads(false)
.SetAllowMmapWrites(false)
.SetArenaBlockSize(arenaBlockSize)
.SetBaseBackgroundCompactions(baseBackgroundCompactions)
.SetBloomLocality(bloomLocality)
.SetBytesPerSync(bytesPerSync)
// .SetCompactionFilter(IntPtr.Zero)
// .SetCompactionFilterFactory(IntPtr.Zero)
.SetCompactionReadaheadSize(1200)
.SetCompactionStyle(RocksDbSharp.Compaction.Level)
//.SetComparator(IntPtr.Zero)
.SetCompression(RocksDbSharp.Compression.Lz4)
.SetCompressionOptions(1, 2, 3, 4)
.SetCompressionPerLevel(new[] { RocksDbSharp.Compression.Lz4 }, 1)
.SetCreateIfMissing()
.SetCreateMissingColumnFamilies()
.SetDbLogDir("test")
.SetDbWriteBufferSize(1000)
.SetDeleteObsoleteFilesPeriodMicros(50)
.SetDisableAutoCompactions(1)
.SetEnableWriteThreadAdaptiveYield(true)
//.SetEnv(IntPtr.Zero)
.SetErrorIfExists()
//.SetFifoCompactionOptions(IntPtr.Zero)
.SetHardPendingCompactionBytesLimit(1)
.SetHardRateLimit(40597)
.SetHashLinkListRep(12)
.SetHashSkipListRep(56, 4, 2)
//.SetInfoLog(IntPtr.Zero)
.SetInfoLogLevel(RocksLogLevel.NUM_INFO_LOG)
.SetInplaceUpdateNumLocks(134)
.SetIsFdCloseOnExec(false)
.SetKeepLogFileNum(1)
.SetLevel0FileNumCompactionTrigger(14)
.SetLevel0SlowdownWritesTrigger(144)
.SetInplaceUpdateSupport(true)
.SetLevel0StopWritesTrigger(24)
.SetLevelCompactionDynamicLevelBytes(true)
.SetLogFileTimeToRoll(154)
.SetManifestPreallocationSize(153)
.SetMaxBackgroundFlushes(3)
.SetMaxBytesForLevelBase(1453)
.SetMaxBytesForLevelMultiplier(2)
.SetMaxBytesForLevelMultiplierAdditional(new[] { 1 }, 1)
.SetMaxCompactionBytes(345678)
.SetMaxFileOpeningThreads(2)
.SetMaxLogFileSize(1)
.SetMaxManifestFileSize(131)
.SetMaxMemCompactionLevel(12)
.SetMaxOpenFiles(20)
.SetMaxSequentialSkipInIterations(13)
.SetMaxSuccessiveMerges(12)
.SetMaxTotalWalSize(12)
.SetMaxWriteBufferNumber(15543)
.SetMaxWriteBufferNumberToMaintain(126)
.SetMemtableHugePageSize(64317)
.SetMemtableHugePageSize(64317)
.SetMemtablePrefixBloomSizeRatio(12)
//.SetMergeOperator(IntPtr.Zero)
.SetMinLevelToCompress(1)
.SetMemtableVectorRep()
.SetMinWriteBufferNumberToMerge(1312)
.SetNumLevels(1)
.SetOptimizeFiltersForHits(56)
.SetParanoidChecks()
.SetSkipLogErrorOnRecovery(true)
.SetSoftPendingCompactionBytesLimit(134)
.SetSoftRateLimit(131)
.SetStatsDumpPeriodSec(532)
.SetTableCacheNumShardbits(12)
.SetTableCacheRemoveScanCountLimit(66)
.SetTargetFileSizeBase(6)
.SetTargetFileSizeMultiplier(2)
.SetUint64addMergeOperator()
//.SetUniversalCompactionOptions(IntPtr.Zero)
.SetUseAdaptiveMutex(false)
.SetUseDirectIoForFlushAndCompaction(true)
.SetUseDirectReads(true)
.SetUseFsync(1)
.SetWalRecoveryMode(RocksDbSharp.Recovery.SkipAnyCorruptedRecords)
.SetWALSizeLimitMB(40)
.SetWALTtlSeconds(1454151413)
.SetWriteBufferSize(45678976543)
.SetPrefixExtractor(SliceTransform.CreateNoOp())
.SetPurgeRedundantKvsWhileFlush(false)
.SetRateLimitDelayMaxMilliseconds(1762)
.SetRecycleLogFileNum(1)
.SetReportBgIoStats(true)
.SetPlainTableFactory(1, 23, 4, 2)
.SetMaxBackgroundCompactions(1);
};
id = new TaskId {
Id = 0, Partition = 0
};
partition = new TopicPartition("source", 0);
stateManager = new ProcessorStateManager(
id,
new List <TopicPartition> {
partition
},
null,
new MockChangelogRegister(),
new MockOffsetCheckpointManager());
task = new Mock <AbstractTask>();
task.Setup(k => k.Id).Returns(id);
context = new ProcessorContext(task.Object, config, stateManager, new StreamMetricsRegistry());
store = new RocksDbKeyValueStore("test-store");
store.Init(context, store);
}