public void WindowStateStoreExist() { var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test-config"; var builder = new StreamBuilder(); builder.Stream <string, string>("test") .GroupByKey() .WindowedBy(TumblingWindowOptions.Of(2000)) .Count(InMemoryWindows <string, long> .As("store")); var driver = new TaskSynchronousTopologyDriver("client", builder.Build().Builder, config, config, default); var input = driver.CreateInputTopic("test", new StringSerDes(), new StringSerDes()); var store = driver.GetStateStore <string, string>("store"); Assert.IsNotNull(store); Assert.IsInstanceOf <TimestampedWindowStore <string, long> >(store); DateTime dt = DateTime.Now; input.PipeInput("coucou", "1"); Assert.AreEqual(1, ((TimestampedWindowStore <string, long>)store).All().ToList().Count); var it = ((TimestampedWindowStore <string, long>)store).Fetch("coucou", dt.AddSeconds(-5), dt.AddSeconds(5)); Assert.IsTrue(it.MoveNext()); Assert.AreEqual(1, it.Current.Value.Value.Value); driver.Dispose(); }
public void TimeWindowingAggNothing() { var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test-window-stream"; var builder = new StreamBuilder(); builder .Stream <string, string>("topic") .GroupByKey() .WindowedBy(TumblingWindowOptions.Of(2000)) .Aggregate( () => 0, (k, v, agg) => Math.Max(v.Length, agg), InMemoryWindows <string, int> .As("store").WithValueSerdes <Int32SerDes>()) .ToStream() .To <StringTimeWindowedSerDes, Int32SerDes>("output"); var topology = builder.Build(); using (var driver = new TopologyTestDriver(topology, config)) { var input = driver.CreateInputTopic <string, string>("topic"); var output = driver.CreateOuputTopic("output", TimeSpan.FromSeconds(1), new StringTimeWindowedSerDes(), new Int64SerDes()); var elements = output.ReadKeyValueList().ToList(); Assert.AreEqual(0, elements.Count); } }
public void TestGetWindowStoreIncorrectType() { var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test-window-stream"; var builder = new StreamBuilder(); builder .Stream <string, string>("topic") .GroupByKey() .WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromSeconds(5))) .Count(InMemoryWindows <string, long> .As("count-store")); var topology = builder.Build(); using var driver = new TopologyTestDriver(topology, config); DateTime dt = DateTime.Now; var input = driver.CreateInputTopic <string, string>("topic"); input.PipeInput("test", "1", dt); var store = driver.GetWindowStore <int, long>("count-store"); Assert.IsNull(store); }
public void TimeWindowingQueryStoreAll() { var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test-window-stream"; var builder = new StreamBuilder(); builder .Stream <string, string>("topic") .GroupByKey() .WindowedBy(TumblingWindowOptions.Of(2000)) .Reduce( (v1, v2) => v1.Length > v2.Length ? v1 : v2, InMemoryWindows <string, string> .As("store")) .ToStream() .To <StringTimeWindowedSerDes, StringSerDes>("output"); var topology = builder.Build(); using (var driver = new TopologyTestDriver(topology, config)) { var input = driver.CreateInputTopic <string, string>("topic"); input.PipeInput("test", "1"); input.PipeInput("test", "2567"); input.PipeInput("test", "32"); var store = driver.GetWindowStore <string, string>("store"); var elements = store.All().ToList(); Assert.AreEqual(1, elements.Count); Assert.AreEqual("test", elements[0].Key.Key); Assert.AreEqual((long)TimeSpan.FromSeconds(2).TotalMilliseconds, elements[0].Key.Window.EndMs - elements[0].Key.Window.StartMs); Assert.AreEqual("2567", elements[0].Value); } }
public void TimeWindowingReduceKeySerdesUnknow() { var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test-window-stream"; var builder = new StreamBuilder(); builder .Stream <string, string>("topic") .GroupByKey() .WindowedBy(TumblingWindowOptions.Of(2000)) .Reduce( (v1, v2) => v1.Length > v2.Length ? v1 : v2, InMemoryWindows <string, string> .As("store")) .ToStream() .To("output"); var topology = builder.Build(); Assert.Throws <StreamsException>(() => { using (var driver = new TopologyTestDriver(topology, config)) { var input = driver.CreateInputTopic <string, string>("topic"); input.PipeInput("test", "1"); } }); }
public void TimeWindowingReduceNothing() { var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test-window-stream"; var builder = new StreamBuilder(); builder .Stream <string, string>("topic") .GroupByKey() .WindowedBy(TumblingWindowOptions.Of(2000)) .Reduce( (v1, v2) => v1.Length > v2.Length ? v1 : v2, InMemoryWindows <string, string> .As("store")) .ToStream() .To <StringTimeWindowedSerDes, StringSerDes>("output"); var topology = builder.Build(); using (var driver = new TopologyTestDriver(topology, config)) { var input = driver.CreateInputTopic <string, string>("topic"); var output = driver.CreateOuputTopic("output", TimeSpan.FromSeconds(1), new StringTimeWindowedSerDes(), new Int64SerDes()); var elements = output.ReadKeyValueList().ToList(); Assert.AreEqual(0, elements.Count); } }
private static void Main(string[] args) { var config = new StreamConfig <StringSerDes, StringSerDes> { ApplicationId = "test-test-driver-app" }; StreamBuilder builder = new StreamBuilder(); var table = builder .Stream <string, string>("test") .GroupByKey() .WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromSeconds(10))) .Count(InMemoryWindows <string, long> .As("count-store")); Topology t = builder.Build(); using (var driver = new TopologyTestDriver(t, config)) { DateTime dt = DateTime.Now; var inputTopic = driver.CreateInputTopic <string, string>("test"); inputTopic.PipeInput("renault", "clio", dt); inputTopic.PipeInput("renault", "megane", dt.AddMilliseconds(10)); Thread.Sleep((int)TimeSpan.FromSeconds(10).TotalMilliseconds); inputTopic.PipeInput("renault", "scenic", dt.AddSeconds(1)); var store = driver.GetWindowStore <string, long>("count-store"); var elements = store.All().ToList(); } }
public void TimeWindowingQueryStoreAll() { var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test-window-stream"; var builder = new StreamBuilder(); builder .Stream <string, string>("topic") .GroupByKey() .WindowedBy(TumblingWindowOptions.Of(2000)) .Aggregate( () => 0, (k, v, agg) => Math.Max(v.Length, agg), InMemoryWindows <string, int> .As("store").WithValueSerdes <Int32SerDes>()); var topology = builder.Build(); using (var driver = new TopologyTestDriver(topology, config)) { var input = driver.CreateInputTopic <string, string>("topic"); input.PipeInput("test", "1"); input.PipeInput("test", "2567"); input.PipeInput("test", "32"); var store = driver.GetWindowStore <string, int>("store"); var elements = store.All().ToList(); Assert.AreEqual(1, elements.Count); Assert.AreEqual("test", elements[0].Key.Key); Assert.AreEqual((long)TimeSpan.FromSeconds(2).TotalMilliseconds, elements[0].Key.Window.EndMs - elements[0].Key.Window.StartMs); Assert.AreEqual(4, elements[0].Value); } }
public void TimeWindowingAggKeySerdesUnknow() { var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test-window-stream"; var builder = new StreamBuilder(); builder .Stream <string, string>("topic") .GroupByKey() .WindowedBy(TumblingWindowOptions.Of(2000)) .Aggregate( () => 0, (k, v, agg) => Math.Max(v.Length, agg), InMemoryWindows <string, int> .As("store").WithValueSerdes <Int32SerDes>()) .ToStream() .To("output"); var topology = builder.Build(); Assert.Throws <StreamsException>(() => { using (var driver = new TopologyTestDriver(topology, config)) { var input = driver.CreateInputTopic <string, string>("topic"); input.PipeInput("test", "1"); } }); }
public void TimeWindowingQueryStore2Window() { var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test-window-stream"; var builder = new StreamBuilder(); builder .Stream <string, string>("topic") .GroupByKey() .WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromSeconds(5))) .Count(InMemoryWindows <string, long> .As("count-store")); var topology = builder.Build(); using (var driver = new TopologyTestDriver(topology, config)) { DateTime dt = DateTime.Now; var input = driver.CreateInputTopic <string, string>("topic"); input.PipeInput("test", "1", dt); input.PipeInput("test", "2", dt); input.PipeInput("test", "3", dt.AddMinutes(1)); var store = driver.GetWindowStore <string, long>("count-store"); var elements = store.All().ToList(); Assert.AreEqual(2, elements.Count); Assert.AreEqual("test", elements[0].Key.Key); Assert.AreEqual((long)TimeSpan.FromSeconds(5).TotalMilliseconds, elements[0].Key.Window.EndMs - elements[0].Key.Window.StartMs); Assert.AreEqual(2, elements[0].Value); Assert.AreEqual("test", elements[1].Key.Key); Assert.AreEqual((long)TimeSpan.FromSeconds(5).TotalMilliseconds, elements[1].Key.Window.EndMs - elements[1].Key.Window.StartMs); Assert.AreEqual(1, elements[1].Value); } }
private static void Main(string[] args) { CancellationTokenSource source = new CancellationTokenSource(); var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test-app"; config.BootstrapServers = "192.168.56.1:9092"; config.SaslMechanism = SaslMechanism.Plain; config.SaslUsername = "******"; config.SaslPassword = "******"; config.SecurityProtocol = SecurityProtocol.SaslPlaintext; config.AutoOffsetReset = AutoOffsetReset.Earliest; config.NumStreamThreads = 1; StreamBuilder builder = new StreamBuilder(); builder.Stream <string, string>("test") .GroupByKey() .WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromMinutes(1))) .Count(InMemoryWindows <string, long> .As("store")) .ToStream() .Map((k, v) => KeyValuePair.Create(k.Key, v.ToString())) .To("output"); Topology t = builder.Build(); //bool taskStart = false; KafkaStream stream = new KafkaStream(t, config); // Subscribe state changed //stream.StateChanged += (old, @new) => //{ // if (!taskStart && @new == KafkaStream.State.RUNNING) // If new state is running, we can quering state store. // { // Task.Factory.StartNew(() => // { // while (!source.Token.IsCancellationRequested) // { // var store = stream.Store(StoreQueryParameters.FromNameAndType("store", QueryableStoreTypes.WindowStore<string, long>())); // var items = store.All().ToList(); // Thread.Sleep(500); // } // }, source.Token); // taskStart = true; // } //}; Console.CancelKeyPress += (o, e) => { source.Cancel(); stream.Close(); }; stream.Start(source.Token); }
/// <summary> /// Materialize a <see cref="InMemoryWindowStore"/> with the given name. /// </summary> /// <typeparam name="KS">New serializer for <typeparamref name="K"/> type</typeparam> /// <typeparam name="VS">New serializer for <typeparamref name="V"/> type</typeparam> /// <param name="storeName">the name of the underlying <see cref="IKTable{K, V}"/> state store; valid characters are ASCII alphanumerics, '.', '_' and '-'.</param> /// <param name="windowSize">the windows size aggregation</param> /// <returns>a new <see cref="InMemoryWindows{K, V}"/> instance with the given storeName</returns> public static InMemoryWindows <K, V> @As <KS, VS>(string storeName, TimeSpan?windowSize) where KS : ISerDes <K>, new() where VS : ISerDes <V>, new() { var m = new InMemoryWindows <K, V>(storeName, new InMemoryWindowStoreSupplier(storeName, TimeSpan.FromDays(1), windowSize.HasValue ? (long)windowSize.Value.TotalMilliseconds : (long?)null)) { KeySerdes = new KS(), ValueSerdes = new VS() }; return(m); }
public async Task GetWindowStateStore() { var timeout = TimeSpan.FromSeconds(10); bool isRunningState = false; DateTime dt = DateTime.Now; var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test"; config.BootstrapServers = "127.0.0.1"; var builder = new StreamBuilder(); builder .Stream <string, string>("test") .GroupByKey() .WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromMinutes(1))) .Count(InMemoryWindows <string, long> .As("store")); var t = builder.Build(); var stream = new KafkaStream(t, config, new SyncKafkaSupplier()); stream.StateChanged += (old, @new) => { if (@new.Equals(KafkaStream.State.RUNNING)) { isRunningState = true; } }; await stream.StartAsync(); while (!isRunningState) { Thread.Sleep(250); if (DateTime.Now > dt + timeout) { break; } } Assert.IsTrue(isRunningState); if (isRunningState) { var store = stream.Store(StoreQueryParameters.FromNameAndType("store", QueryableStoreTypes.WindowStore <string, long>())); Assert.IsNotNull(store); } stream.Dispose(); }
public void GetWStateStoreInvalidStateStoreException() { var timeout = TimeSpan.FromSeconds(10); var source = new CancellationTokenSource(); bool state = false; DateTime dt = DateTime.Now; var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test"; config.BootstrapServers = "127.0.0.1"; var builder = new StreamBuilder(); builder .Stream <string, string>("test") .GroupByKey() .WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromMinutes(1))) .Count(InMemoryWindows <string, long> .As("store")); var t = builder.Build(); var stream = new KafkaStream(t, config, new SyncKafkaSupplier()); stream.StateChanged += (old, @new) => { if ([email protected](KafkaStream.State.RUNNING)) { if (!state) { Assert.Throws <InvalidStateStoreException>(() => stream.Store(StoreQueryParameters.FromNameAndType("store", QueryableStoreTypes.WindowStore <string, long>()))); state = true; } } }; stream.Start(source.Token); while (!state) { Thread.Sleep(250); if (DateTime.Now > dt + timeout) { break; } } Assert.IsTrue(state); source.Cancel(); stream.Close(); }
public void TimeWindowingAggWithMaterialize() { var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test-window-stream"; var builder = new StreamBuilder(); builder .Stream <string, string>("topic") .GroupByKey() .WindowedBy(TumblingWindowOptions.Of(2000)) .Aggregate( () => 0, (k, v, agg) => Math.Max(v.Length, agg), InMemoryWindows <string, int> .As("store").WithValueSerdes <Int32SerDes>()) .ToStream() .To <StringTimeWindowedSerDes, Int32SerDes>("output"); var topology = builder.Build(); using (var driver = new TopologyTestDriver(topology, config)) { var input = driver.CreateInputTopic <string, string>("topic"); var output = driver.CreateOuputTopic("output", TimeSpan.FromSeconds(1), new StringTimeWindowedSerDes(), new Int32SerDes()); input.PipeInput("test", "1"); input.PipeInput("test", "230"); input.PipeInput("test", "32"); var elements = output.ReadKeyValueList().ToList(); Assert.AreEqual(3, elements.Count); Assert.AreEqual("test", elements[0].Message.Key.Key); Assert.AreEqual((long)TimeSpan.FromSeconds(10).TotalMilliseconds, elements[0].Message.Key.Window.EndMs - elements[0].Message.Key.Window.StartMs); Assert.AreEqual(1, elements[0].Message.Value); Assert.AreEqual("test", elements[1].Message.Key.Key); Assert.AreEqual(elements[0].Message.Key.Window, elements[1].Message.Key.Window); Assert.AreEqual(3, elements[1].Message.Value); Assert.AreEqual("test", elements[2].Message.Key.Key); Assert.AreEqual(elements[0].Message.Key.Window, elements[2].Message.Key.Window); Assert.AreEqual(3, elements[2].Message.Value); } }
public void WindowStateStoreNotExist() { var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test-config"; var builder = new StreamBuilder(); builder.Stream <string, string>("test") .GroupByKey() .WindowedBy(TumblingWindowOptions.Of(2000)) .Count(InMemoryWindows <string, long> .As("store")); var driver = new TaskSynchronousTopologyDriver("client", builder.Build().Builder, config, config, default); driver.CreateInputTopic("test", new StringSerDes(), new StringSerDes()); var store = driver.GetStateStore <string, string>("store2"); Assert.IsNull(store); driver.Dispose(); }
public async Task GetWindowElementInStateStore() { var timeout = TimeSpan.FromSeconds(10); bool isRunningState = false; DateTime dt = DateTime.Now; var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test"; config.BootstrapServers = "127.0.0.1"; config.PollMs = 10; var supplier = new SyncKafkaSupplier(); var producer = supplier.GetProducer(config.ToProducerConfig()); var builder = new StreamBuilder(); builder .Stream <string, string>("test") .GroupByKey() .WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromMinutes(1))) .Count(InMemoryWindows <string, long> .As("store")); var t = builder.Build(); var stream = new KafkaStream(t, config, supplier); stream.StateChanged += (old, @new) => { if (@new.Equals(KafkaStream.State.RUNNING)) { isRunningState = true; } }; await stream.StartAsync(); while (!isRunningState) { Thread.Sleep(250); if (DateTime.Now > dt + timeout) { break; } } Assert.IsTrue(isRunningState); if (isRunningState) { var serdes = new StringSerDes(); dt = DateTime.Now; producer.Produce("test", new Confluent.Kafka.Message <byte[], byte[]> { Key = serdes.Serialize("key1", new SerializationContext()), Value = serdes.Serialize("coucou", new SerializationContext()), Timestamp = new Confluent.Kafka.Timestamp(dt) }); Thread.Sleep(50); var store = stream.Store(StoreQueryParameters.FromNameAndType("store", QueryableStoreTypes.WindowStore <string, long>())); Assert.IsNotNull(store); var @enum = store.All(); Assert.AreEqual(1, store.All().ToList().Count); var item = store.Fetch("key1", dt.AddMinutes(-1), dt.AddMinutes(1)); Assert.IsNotNull(item); Assert.IsTrue(item.MoveNext()); Assert.IsTrue(item.Current.HasValue); Assert.AreEqual(1, item.Current.Value.Value); item.Dispose(); } stream.Dispose(); }