public void WithNullSerDes() { // WITH NULL SERDES, in running KeySerdes must be StringSerdes, and ValueSerdes Int64SerDes var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test-agg"; var builder = new StreamBuilder(); Materialized <string, long, IKeyValueStore <Bytes, byte[]> > m = Materialized <string, long, IKeyValueStore <Bytes, byte[]> > .Create("agg-store") .With(null, null); builder .Stream <string, string>("topic") .GroupByKey() .Aggregate(() => 0, (k, v, agg) => agg + 1, m) .ToStream() .To("output-topic"); var topology = builder.Build(); Assert.Throws <StreamsException>(() => { using (var driver = new TopologyTestDriver(topology, config)) { var input = driver.CreateInputTopic <string, string>("topic"); input.PipeInput("test", "1"); } }); }
public void TestWithKeySerdes() { var m = Materialized <string, string, IStateStore> .Create("test").WithKeySerdes(new StringSerDes()); Assert.IsNotNull(m.KeySerdes); Assert.IsAssignableFrom <StringSerDes>(m.KeySerdes); }
public void TestWithValueSerdes2() { var m = Materialized <string, string, IStateStore> .Create("test").WithValueSerdes <StringSerDes>(); Assert.IsNotNull(m.ValueSerdes); Assert.IsAssignableFrom <StringSerDes>(m.ValueSerdes); }
public IKTable <Windowed <K>, VR> Aggregate <VR>(Initializer <VR> initializer, Aggregator <K, V, VR> aggregator, Materialized <K, VR, WindowStore <Bytes, byte[]> > materialized, string named = null) { CheckIfParamNull(initializer, "initializer"); CheckIfParamNull(aggregator, "aggregator"); materialized = materialized ?? Materialized <K, VR, WindowStore <Bytes, byte[]> > .Create(); if (materialized.KeySerdes == null) { materialized.WithKeySerdes(KeySerdes); } string name = new Named(named).OrElseGenerateWithPrefix(builder, KGroupedStream.AGGREGATE_NAME); materialized.UseProvider(builder, KGroupedStream.AGGREGATE_NAME); var aggSupplier = new KStreamWindowAggregate <K, V, VR, W>( windowOptions, materialized.StoreName, initializer, aggregator); ISerDes <Windowed <K> > windowSerdes = materialized.KeySerdes != null ? new TimeWindowedSerDes <K>(materialized.KeySerdes, windowOptions.Size) : null; return(aggBuilder.BuildWindow(name, new TimestampedWindowStoreMaterializer <K, VR, W>(windowOptions, materialized).Materialize(), aggSupplier, materialized.QueryableStoreName, windowSerdes, materialized.ValueSerdes)); }
public void StoresPersistentKeyValueStoreTest() { var builder = new StreamBuilder(); // Same like that : // builder.Table("table-topic", RocksDb<string, string>.As("table-topic-store")); builder.Table("table-topic", Materialized <string, string, IKeyValueStore <Bytes, byte[]> > .Create( Streamiz.Kafka.Net.State.Stores.PersistentKeyValueStore("table-topic-store"))); var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test-map"; config.UseRandomRocksDbConfigForTest(); Topology t = builder.Build(); using (var driver = new TopologyTestDriver(t, config)) { var inputTopic = driver.CreateInputTopic <string, string>("table-topic"); inputTopic.PipeInput("key1", "1"); inputTopic.PipeInput("key2", "2"); var store = driver.GetKeyValueStore <string, string>("table-topic-store"); Assert.IsNotNull(store); var resultK1 = store.Get("key1"); var resultK2 = store.Get("key2"); Assert.AreEqual("1", resultK1); Assert.AreEqual("2", resultK2); } config.RemoveRocksDbFolderForTest(); }
public void TimeWindowingCountKeySerdesUnknow() { var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test-window-stream"; var builder = new StreamBuilder(); builder .Stream <string, string>("topic") .GroupByKey() .WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromSeconds(10))) .Count(Materialized <string, long, WindowStore <Bytes, byte[]> > .Create("count-store")) .ToStream() .To("output"); var topology = builder.Build(); Assert.Throws <StreamsException>(() => { using (var driver = new TopologyTestDriver(topology, config)) { var input = driver.CreateInputTopic <string, string>("topic"); input.PipeInput("test", "1"); } }); }
public void WithNullValueSerDes() { // WITH VALUE NULL SERDES, in running KeySerdes must be StringSerdes, and ValueSerdes Int64SerDes var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test-window-count"; var builder = new StreamBuilder(); Materialized <string, string, IWindowStore <Bytes, byte[]> > m = Materialized <string, string, IWindowStore <Bytes, byte[]> > .Create("store") .With(null, null); builder .Stream <string, string>("topic") .GroupByKey() .WindowedBy(TumblingWindowOptions.Of(2000)) .Reduce((v1, v2) => v1.Length > v2.Length ? v1 : v2, m) .ToStream() .To("output-topic"); var topology = builder.Build(); Assert.Throws <StreamsException>(() => { using (var driver = new TopologyTestDriver(topology, config)) { var input = driver.CreateInputTopic <string, string>("topic"); var output = driver.CreateOuputTopic("output-topic", TimeSpan.FromSeconds(1), new StringTimeWindowedSerDes(), new Int64SerDes()); input.PipeInput("test", "1"); } }); }
public void WithNullSerDes() { // WITH NULL SERDES, in running KeySerdes must be StringSerdes, and ValueSerdes Int64SerDes var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test-reduce"; var builder = new StreamBuilder(); Materialized <string, int, IKeyValueStore <Bytes, byte[]> > m = Materialized <string, int, IKeyValueStore <Bytes, byte[]> > .Create("reduce-store") .With(null, null); builder .Table <string, string>("topic") .MapValues((v) => v.Length) .GroupBy((k, v) => KeyValuePair.Create(k.ToUpper(), v)) .Reduce((v1, v2) => Math.Max(v1, v2), (v1, v2) => v2, m) .ToStream() .To("output-topic"); var topology = builder.Build(); Assert.Throws <StreamsException>(() => { using (var driver = new TopologyTestDriver(topology, config)) { var input = driver.CreateInputTopic <string, string>("topic"); input.PipeInput("test", "1"); } }); }
public void TestCreateKeyValueBytesStoreSupplier() { var m = Materialized <string, string, IKeyValueStore <Bytes, byte[]> > .Create(new InMemoryKeyValueBytesStoreSupplier("name")); Assert.IsNotNull(m); Assert.IsNotNull(m.StoreSupplier); }
public void TestStoreName1() { var m = Materialized <string, string, IStateStore> .Create("test"); Assert.IsNotNull(m); Assert.AreEqual("test", m.StoreName); Assert.IsNull(m.QueryableStoreName); }
public void TestWithCachingDisabled() { var m = Materialized <string, string, IStateStore> .Create("test") .WithCachingDisabled(); Assert.IsNotNull(m); Assert.IsFalse(m.CachingEnabled); }
public void TestWithRetention() { var m = Materialized <string, string, IStateStore> .Create("test") .WithRetention(TimeSpan.FromSeconds(10)); Assert.IsNotNull(m); Assert.AreEqual(TimeSpan.FromSeconds(10), m.Retention); }
public void TestStoreName3() { var m = Materialized <string, string, IStateStore> .Create() .UseProvider(null, ""); Assert.IsNotNull(m); Assert.AreEqual(string.Empty, m.StoreName); Assert.IsNull(m.QueryableStoreName); }
/// <summary> /// Create a <see cref="IGlobalKTable{K,V}"/> for the specified topic. /// Input keyvalue records with <code>null</code> key will be dropped. /// The resulting <see cref="IGlobalKTable{K,V}"/> will be materialized in a local <see cref="IKeyValueStore{K, V}"/> using the given /// <see cref="Materialized{K, V, S}"/> instance. /// However, no internal changelog topic is created since the original input topic can be used for recovery. /// Note that <see cref="IGlobalKTable{K,V}"/> always applies <code>"auto.offset.reset"</code> strategy <code>"earliest"</code> /// regardless of the specified value in <see cref="IStreamConfig"/>. /// </summary> /// <typeparam name="K">Key type of record</typeparam> /// <typeparam name="V">Value type of record</typeparam> /// <param name="topic">the topic name, can't be null</param> /// <param name="keySerdes">Key deserializer</param> /// <param name="valueSerdes">Value deserializer</param> /// <param name="materialized">the instance of <see cref="Materialized{K, V, S}"/> used to materialize a state store.</param> /// <param name="named">Processor name</param> /// <param name="extractor">the timestamp extractor to be used. If null the default timestamp extractor from config will be used</param> /// <returns>a <see cref="IGlobalKTable{K,V}"/> for the specified topic</returns> /// <exception cref="ArgumentException">Throw <see cref="ArgumentException"/> if topic is null or empty</exception> public IGlobalKTable <K, V> GlobalTable <K, V>(string topic, ISerDes <K> keySerdes, ISerDes <V> valueSerdes, Materialized <K, V, IKeyValueStore <Bytes, byte[]> > materialized, string named, ITimestampExtractor extractor) { materialized = materialized ?? Materialized <K, V, IKeyValueStore <Bytes, byte[]> > .Create(); var consumedInternal = new ConsumedInternal <K, V>(named, keySerdes, valueSerdes, extractor); materialized.UseProvider(internalStreamBuilder, $"{topic}-")?.InitConsumed(consumedInternal); return(internalStreamBuilder.GlobalTable(topic, consumedInternal, materialized)); }
public void TestStoreName5() { var m = Materialized <string, string, IStateStore> .Create("store") .UseProvider(new MyNameProvider(), "pref"); Assert.IsNotNull(m); Assert.AreEqual("store", m.StoreName); Assert.AreEqual("store", m.QueryableStoreName); }
public void TestCreateWithSerdes() { var m = Materialized <string, string, IKeyValueStore <Bytes, byte[]> > .Create <StringSerDes, StringSerDes>(); Assert.IsNotNull(m); Assert.IsNotNull(m.KeySerdes); Assert.IsNotNull(m.ValueSerdes); Assert.IsAssignableFrom <StringSerDes>(m.KeySerdes); Assert.IsAssignableFrom <StringSerDes>(m.ValueSerdes); }
public void TestCreateKeyValueBytesStoreSupplierWithSerdes() { var m = Materialized <string, string, IKeyValueStore <Bytes, byte[]> > .Create <StringSerDes, StringSerDes>(new InMemoryKeyValueBytesStoreSupplier("name")); Assert.IsNotNull(m); Assert.IsNotNull(m.StoreSupplier); Assert.IsNotNull(m.KeySerdes); Assert.IsNotNull(m.ValueSerdes); Assert.IsAssignableFrom <StringSerDes>(m.KeySerdes); Assert.IsAssignableFrom <StringSerDes>(m.ValueSerdes); }
public void TestWithLoggingEnabled() { var topicConfig = new Dictionary <string, string> { { "test", "test" } }; var m = Materialized <string, string, IStateStore> .Create("test") .WithLoggingEnabled(topicConfig); Assert.IsNotNull(m); Assert.IsTrue(m.LoggingEnabled); Assert.AreEqual(topicConfig, m.TopicConfig); }
/// <summary> /// Create a <see cref="IKTable{K, V}"/> for the specified topic. /// Input keyvalue records with null key will be dropped. /// /// Note that the specified input topic must be partitioned by key. /// If this is not the case the returned <see cref="IKTable{K, V}"/> will be corrupted. /// /// The resulting <see cref="IKTable{K, V}"/> will be materialized in a local <see cref="IKeyValueStore{K, V}"/> using the given /// <see cref="Materialized{K, V, S}"/> instance. /// </summary> /// <typeparam name="K">Key type of record</typeparam> /// <typeparam name="V">Value type of record</typeparam> /// <param name="topic">the topic name, can't be null</param> /// <param name="keySerdes">Key deserializer</param> /// <param name="valueSerdes">Value deserializer</param> /// <param name="materialized">the instance of <see cref="Materialized{K, V, S}"/> used to materialize a state store.</param> /// <param name="named">Processor name</param> /// <param name="extractor">the timestamp extractor to be used. If null the default timestamp extractor from config will be used</param> /// <returns>a <see cref="IKTable{K, V}"/> for the specified topic</returns> /// <exception cref="ArgumentException">Throw <see cref="ArgumentException"/> if topic is null or empty</exception> public IKTable <K, V> Table <K, V>(string topic, ISerDes <K> keySerdes, ISerDes <V> valueSerdes, Materialized <K, V, IKeyValueStore <Bytes, byte[]> > materialized, string named, ITimestampExtractor extractor) { if (string.IsNullOrEmpty(topic)) { throw new ArgumentException("Topic of KTable must not be null or empty"); } materialized = materialized ?? Materialized <K, V, IKeyValueStore <Bytes, byte[]> > .Create(); var consumedInternal = new ConsumedInternal <K, V>(named, keySerdes, valueSerdes, extractor); materialized.UseProvider(internalStreamBuilder, $"{topic}-").InitConsumed(consumedInternal); return(internalStreamBuilder.Table(topic, consumedInternal, materialized)); }
public void WithNullSerDes() { // WITH NULL SERDES, in running KeySerdes must be StringSerdes, and ValueSerdes Int64SerDes var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test-count"; var builder = new StreamBuilder(); Materialized <string, long, IKeyValueStore <Bytes, byte[]> > m = Materialized <string, long, IKeyValueStore <Bytes, byte[]> > .Create("count-store") .With(null, null); builder .Table <string, string>("topic") .GroupBy((k, v) => KeyValuePair.Create(k.ToUpper(), v)) .Count(m) .ToStream() .To("output-topic"); var topology = builder.Build(); using (var driver = new TopologyTestDriver(topology, config)) { var input = driver.CreateInputTopic <string, string>("topic"); var output = driver.CreateOuputTopic <string, long, StringSerDes, Int64SerDes>("output-topic"); input.PipeInput("test", "1"); input.PipeInput("test", "30"); IEnumerable <KeyValuePair <string, long> > expected = new List <KeyValuePair <string, long> > { KeyValuePair.Create("TEST", 1L), KeyValuePair.Create("TEST", 0L), KeyValuePair.Create("TEST", 1L) }; var records = output.ReadKeyValueList().Select(r => KeyValuePair.Create(r.Message.Key, r.Message.Value)).ToList(); Assert.AreEqual(expected, records); } }
public void WithNullReducer() { // WITH NULL SERDES, in running KeySerdes must be StringSerdes, and ValueSerdes Int64SerDes var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test-reduce"; var builder = new StreamBuilder(); Materialized <string, int, IKeyValueStore <Bytes, byte[]> > m = Materialized <string, int, IKeyValueStore <Bytes, byte[]> > .Create("reduce-store") .With(null, null); Assert.Throws <ArgumentNullException>(() => { builder .Table <string, string>("topic") .MapValues((v) => v.Length) .GroupBy((k, v) => KeyValuePair.Create(k.ToUpper(), v)) .Reduce((Reducer <int>)null, (Reducer <int>)null, m); }); }
public void WithNullValueSerDes() { // WITH VALUE NULL SERDES, in running KeySerdes must be StringSerdes, and ValueSerdes Int64SerDes var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test-window-count"; var builder = new StreamBuilder(); Materialized <string, long, WindowStore <Bytes, byte[]> > m = Materialized <string, long, WindowStore <Bytes, byte[]> > .Create("count-store") .With(null, null); builder .Stream <string, string>("topic") .GroupByKey() .WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromSeconds(5))) .Count(m) .ToStream() .To <StringTimeWindowedSerDes, Int64SerDes>("output-topic"); var topology = builder.Build(); using (var driver = new TopologyTestDriver(topology, config)) { var input = driver.CreateInputTopic <string, string>("topic"); var output = driver.CreateOuputTopic("output-topic", TimeSpan.FromSeconds(1), new StringTimeWindowedSerDes(), new Int64SerDes()); input.PipeInput("test", "1"); input.PipeInput("test-test", "30"); var records = output.ReadKeyValueList().ToList(); Assert.AreEqual(2, records.Count); Assert.AreEqual("test", records[0].Message.Key.Key); Assert.AreEqual(1, records[0].Message.Value); Assert.AreEqual("test-test", records[1].Message.Key.Key); Assert.AreEqual(1, records[1].Message.Value); Assert.AreEqual(records[0].Message.Key.Window, records[1].Message.Key.Window); } }
public void TimeWindowingCountWithMaterialize() { var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test-window-stream"; var builder = new StreamBuilder(); builder .Stream <string, string>("topic") .GroupByKey() .WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromSeconds(10))) .Count(Materialized <string, long, WindowStore <Bytes, byte[]> > .Create("count-store")) .ToStream() .To <StringTimeWindowedSerDes, Int64SerDes>("output"); var topology = builder.Build(); using (var driver = new TopologyTestDriver(topology, config)) { var input = driver.CreateInputTopic <string, string>("topic"); var output = driver.CreateOuputTopic("output", TimeSpan.FromSeconds(1), new StringTimeWindowedSerDes(), new Int64SerDes()); input.PipeInput("test", "1"); input.PipeInput("test", "2"); input.PipeInput("test", "3"); var elements = output.ReadKeyValueList().ToList(); Assert.AreEqual(3, elements.Count); Assert.AreEqual("test", elements[0].Message.Key.Key); Assert.AreEqual((long)TimeSpan.FromSeconds(10).TotalMilliseconds, elements[0].Message.Key.Window.EndMs - elements[0].Message.Key.Window.StartMs); Assert.AreEqual(1, elements[0].Message.Value); Assert.AreEqual("test", elements[1].Message.Key.Key); Assert.AreEqual(elements[0].Message.Key.Window, elements[1].Message.Key.Window); Assert.AreEqual(2, elements[1].Message.Value); Assert.AreEqual("test", elements[2].Message.Key.Key); Assert.AreEqual(elements[0].Message.Key.Window, elements[2].Message.Key.Window); Assert.AreEqual(3, elements[2].Message.Value); } }
public void StoresPersistentKeyValueStoreTest() { var builder = new StreamBuilder(); builder.Stream <string, string>("topic") .GroupByKey() .WindowedBy(TumblingWindowOptions.Of(1000)) .Count( Materialized <string, long, IWindowStore <Bytes, byte[]> > .Create( Streamiz.Kafka.Net.State.Stores.PersistentWindowStore( "rocksdb-w-store", TimeSpan.FromDays(1), TimeSpan.FromSeconds(1)))); var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test-rocksdb-window-store"; config.UseRandomRocksDbConfigForTest(); Topology t = builder.Build(); using (var driver = new TopologyTestDriver(t, config)) { DateTime dt = DateTime.Now; var inputTopic = driver.CreateInputTopic <string, string>("topic"); inputTopic.PipeInput("abc", "1", dt); inputTopic.PipeInput("key1", "1", dt); inputTopic.PipeInput("test", "1", dt); var store = driver.GetWindowStore <string, long>("rocksdb-w-store"); Assert.IsNotNull(store); var k1 = store.FetchAll(dt.AddMinutes(-10), dt.AddMinutes(10)).ToList(); Assert.AreEqual(3, k1.Count); } config.RemoveRocksDbFolderForTest(); }
public IKTable <Windowed <K>, V> Reduce(Reducer <V> reducer, Materialized <K, V, WindowStore <Bytes, byte[]> > materialized, string named = null) { CheckIfParamNull(reducer, "reducer"); materialized = materialized ?? Materialized <K, V, WindowStore <Bytes, byte[]> > .Create(); if (materialized.KeySerdes == null) { materialized.WithKeySerdes(KeySerdes); } if (materialized.ValueSerdes == null) { materialized.WithValueSerdes(ValueSerdes); } string name = new Named(named).OrElseGenerateWithPrefix(builder, KGroupedStream.REDUCE_NAME); materialized.UseProvider(builder, KGroupedStream.REDUCE_NAME); var aggSupplier = new KStreamWindowAggregate <K, V, V, W>( windowOptions, materialized.StoreName, () => default,
public IKTable <Windowed <K>, long> Count(string named) => Count(Materialized <K, long, WindowStore <Bytes, byte[]> > .Create(), named);
public IKTable <Windowed <K>, long> Count(Materialized <K, long, WindowStore <Bytes, byte[]> > materialized, string named = null) { materialized = materialized ?? Materialized <K, long, WindowStore <Bytes, byte[]> > .Create(); return(DoCount(materialized, named)); }
public IKTable <Windowed <K>, VR> Aggregate <VR, VRS>(Initializer <VR> initializer, Aggregator <K, V, VR> aggregator) where VRS : ISerDes <VR>, new() { var materialized = Materialized <K, VR, WindowStore <Bytes, byte[]> > .Create().WithValueSerdes(new VRS()).WithKeySerdes(KeySerdes); return(Aggregate(initializer, aggregator, materialized)); }