Example #1
0
        public void TimeWindowingAggKeySerdesUnknow()
        {
            var config = new StreamConfig <StringSerDes, StringSerDes>();

            config.ApplicationId = "test-window-stream";

            var builder = new StreamBuilder();

            builder
            .Stream <string, string>("topic")
            .GroupByKey()
            .WindowedBy(TumblingWindowOptions.Of(2000))
            .Aggregate(
                () => 0,
                (k, v, agg) => Math.Max(v.Length, agg),
                InMemoryWindows <string, int> .As("store").WithValueSerdes <Int32SerDes>())
            .ToStream()
            .To("output");

            var topology = builder.Build();

            Assert.Throws <StreamsException>(() =>
            {
                using (var driver = new TopologyTestDriver(topology, config))
                {
                    var input = driver.CreateInputTopic <string, string>("topic");
                    input.PipeInput("test", "1");
                }
            });
        }
        public void TimeWindowingReduceKeySerdesUnknow()
        {
            var config = new StreamConfig <StringSerDes, StringSerDes>();

            config.ApplicationId = "test-window-stream";

            var builder = new StreamBuilder();

            builder
            .Stream <string, string>("topic")
            .GroupByKey()
            .WindowedBy(TumblingWindowOptions.Of(2000))
            .Reduce(
                (v1, v2) => v1.Length > v2.Length ? v1 : v2,
                InMemoryWindows <string, string> .As("store"))
            .ToStream()
            .To("output");

            var topology = builder.Build();

            Assert.Throws <StreamsException>(() =>
            {
                using (var driver = new TopologyTestDriver(topology, config))
                {
                    var input = driver.CreateInputTopic <string, string>("topic");
                    input.PipeInput("test", "1");
                }
            });
        }
Example #3
0
        public void TimeWindowingQueryStoreAll()
        {
            var config = new StreamConfig <StringSerDes, StringSerDes>();

            config.ApplicationId = "test-window-stream";

            var builder = new StreamBuilder();

            builder
            .Stream <string, string>("topic")
            .GroupByKey()
            .WindowedBy(TumblingWindowOptions.Of(2000))
            .Aggregate(
                () => 0,
                (k, v, agg) => Math.Max(v.Length, agg),
                InMemoryWindows <string, int> .As("store").WithValueSerdes <Int32SerDes>());

            var topology = builder.Build();

            using (var driver = new TopologyTestDriver(topology, config))
            {
                var input = driver.CreateInputTopic <string, string>("topic");
                input.PipeInput("test", "1");
                input.PipeInput("test", "2567");
                input.PipeInput("test", "32");
                var store    = driver.GetWindowStore <string, int>("store");
                var elements = store.All().ToList();
                Assert.AreEqual(1, elements.Count);
                Assert.AreEqual("test", elements[0].Key.Key);
                Assert.AreEqual((long)TimeSpan.FromSeconds(2).TotalMilliseconds, elements[0].Key.Window.EndMs - elements[0].Key.Window.StartMs);
                Assert.AreEqual(4, elements[0].Value);
            }
        }
Example #4
0
        public void TimeWindowingCountKeySerdesUnknow()
        {
            var config = new StreamConfig <StringSerDes, StringSerDes>();

            config.ApplicationId = "test-window-stream";

            var builder = new StreamBuilder();

            builder
            .Stream <string, string>("topic")
            .GroupByKey()
            .WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromSeconds(10)))
            .Count(Materialized <string, long, WindowStore <Bytes, byte[]> > .Create("count-store"))
            .ToStream()
            .To("output");

            var topology = builder.Build();

            Assert.Throws <StreamsException>(() =>
            {
                using (var driver = new TopologyTestDriver(topology, config))
                {
                    var input = driver.CreateInputTopic <string, string>("topic");
                    input.PipeInput("test", "1");
                }
            });
        }
Example #5
0
        public void TimeWindowingQueryStore2Window()
        {
            var config = new StreamConfig <StringSerDes, StringSerDes>();

            config.ApplicationId = "test-window-stream";

            var builder = new StreamBuilder();

            builder
            .Stream <string, string>("topic")
            .GroupByKey()
            .WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromSeconds(5)))
            .Count(InMemoryWindows <string, long> .As("count-store"));

            var topology = builder.Build();

            using (var driver = new TopologyTestDriver(topology, config))
            {
                DateTime dt    = DateTime.Now;
                var      input = driver.CreateInputTopic <string, string>("topic");
                input.PipeInput("test", "1", dt);
                input.PipeInput("test", "2", dt);
                input.PipeInput("test", "3", dt.AddMinutes(1));
                var store    = driver.GetWindowStore <string, long>("count-store");
                var elements = store.All().ToList();
                Assert.AreEqual(2, elements.Count);
                Assert.AreEqual("test", elements[0].Key.Key);
                Assert.AreEqual((long)TimeSpan.FromSeconds(5).TotalMilliseconds, elements[0].Key.Window.EndMs - elements[0].Key.Window.StartMs);
                Assert.AreEqual(2, elements[0].Value);
                Assert.AreEqual("test", elements[1].Key.Key);
                Assert.AreEqual((long)TimeSpan.FromSeconds(5).TotalMilliseconds, elements[1].Key.Window.EndMs - elements[1].Key.Window.StartMs);
                Assert.AreEqual(1, elements[1].Value);
            }
        }
        public void TimeWindowingQueryStoreAll()
        {
            var config = new StreamConfig <StringSerDes, StringSerDes>();

            config.ApplicationId = "test-window-stream";

            var builder = new StreamBuilder();

            builder
            .Stream <string, string>("topic")
            .GroupByKey()
            .WindowedBy(TumblingWindowOptions.Of(2000))
            .Reduce(
                (v1, v2) => v1.Length > v2.Length ? v1 : v2,
                InMemoryWindows <string, string> .As("store"))
            .ToStream()
            .To <StringTimeWindowedSerDes, StringSerDes>("output");

            var topology = builder.Build();

            using (var driver = new TopologyTestDriver(topology, config))
            {
                var input = driver.CreateInputTopic <string, string>("topic");
                input.PipeInput("test", "1");
                input.PipeInput("test", "2567");
                input.PipeInput("test", "32");
                var store    = driver.GetWindowStore <string, string>("store");
                var elements = store.All().ToList();
                Assert.AreEqual(1, elements.Count);
                Assert.AreEqual("test", elements[0].Key.Key);
                Assert.AreEqual((long)TimeSpan.FromSeconds(2).TotalMilliseconds, elements[0].Key.Window.EndMs - elements[0].Key.Window.StartMs);
                Assert.AreEqual("2567", elements[0].Value);
            }
        }
        public void TimeWindowingReduceNothing()
        {
            var config = new StreamConfig <StringSerDes, StringSerDes>();

            config.ApplicationId = "test-window-stream";

            var builder = new StreamBuilder();

            builder
            .Stream <string, string>("topic")
            .GroupByKey()
            .WindowedBy(TumblingWindowOptions.Of(2000))
            .Reduce(
                (v1, v2) => v1.Length > v2.Length ? v1 : v2,
                InMemoryWindows <string, string> .As("store"))
            .ToStream()
            .To <StringTimeWindowedSerDes, StringSerDes>("output");

            var topology = builder.Build();

            using (var driver = new TopologyTestDriver(topology, config))
            {
                var input    = driver.CreateInputTopic <string, string>("topic");
                var output   = driver.CreateOuputTopic("output", TimeSpan.FromSeconds(1), new StringTimeWindowedSerDes(), new Int64SerDes());
                var elements = output.ReadKeyValueList().ToList();
                Assert.AreEqual(0, elements.Count);
            }
        }
        public void TestGetWindowStoreIncorrectType()
        {
            var config = new StreamConfig <StringSerDes, StringSerDes>();

            config.ApplicationId = "test-window-stream";

            var builder = new StreamBuilder();

            builder
            .Stream <string, string>("topic")
            .GroupByKey()
            .WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromSeconds(5)))
            .Count(InMemoryWindows <string, long> .As("count-store"));

            var topology = builder.Build();

            using var driver = new TopologyTestDriver(topology, config);
            DateTime dt    = DateTime.Now;
            var      input = driver.CreateInputTopic <string, string>("topic");

            input.PipeInput("test", "1", dt);
            var store = driver.GetWindowStore <int, long>("count-store");

            Assert.IsNull(store);
        }
        public void WithNullValueSerDes()
        {
            // WITH VALUE NULL SERDES, in running KeySerdes must be StringSerdes, and ValueSerdes Int64SerDes
            var config = new StreamConfig <StringSerDes, StringSerDes>();

            config.ApplicationId = "test-window-count";

            var builder = new StreamBuilder();

            Materialized <string, string, IWindowStore <Bytes, byte[]> > m =
                Materialized <string, string, IWindowStore <Bytes, byte[]> >
                .Create("store")
                .With(null, null);

            builder
            .Stream <string, string>("topic")
            .GroupByKey()
            .WindowedBy(TumblingWindowOptions.Of(2000))
            .Reduce((v1, v2) => v1.Length > v2.Length ? v1 : v2, m)
            .ToStream()
            .To("output-topic");

            var topology = builder.Build();

            Assert.Throws <StreamsException>(() =>
            {
                using (var driver = new TopologyTestDriver(topology, config))
                {
                    var input  = driver.CreateInputTopic <string, string>("topic");
                    var output = driver.CreateOuputTopic("output-topic", TimeSpan.FromSeconds(1), new StringTimeWindowedSerDes(), new Int64SerDes());
                    input.PipeInput("test", "1");
                }
            });
        }
        public void TimeWindowingCountNothing()
        {
            var config = new StreamConfig <StringSerDes, StringSerDes>();

            config.ApplicationId = "test-window-stream";

            var builder = new StreamBuilder();

            builder
            .Stream <string, string>("topic")
            .GroupByKey()
            .WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromSeconds(1)))
            .Count()
            .ToStream()
            .To <StringTimeWindowedSerDes, Int64SerDes>("output");

            var topology = builder.Build();

            using var driver = new TopologyTestDriver(topology, config);
            var input    = driver.CreateInputTopic <string, string>("topic");
            var output   = driver.CreateOuputTopic("output", TimeSpan.FromSeconds(1), new StringTimeWindowedSerDes(), new Int64SerDes());
            var elements = output.ReadKeyValueList().ToList();

            Assert.AreEqual(0, elements.Count);
        }
Example #11
0
        public void WindowStateStoreExist()
        {
            var config = new StreamConfig <StringSerDes, StringSerDes>();

            config.ApplicationId = "test-config";

            var builder = new StreamBuilder();

            builder.Stream <string, string>("test")
            .GroupByKey()
            .WindowedBy(TumblingWindowOptions.Of(2000))
            .Count(InMemoryWindows <string, long> .As("store"));

            var driver = new TaskSynchronousTopologyDriver("client", builder.Build().Builder, config, config, default);
            var input  = driver.CreateInputTopic("test", new StringSerDes(), new StringSerDes());
            var store  = driver.GetStateStore <string, string>("store");

            Assert.IsNotNull(store);
            Assert.IsInstanceOf <TimestampedWindowStore <string, long> >(store);
            DateTime dt = DateTime.Now;

            input.PipeInput("coucou", "1");

            Assert.AreEqual(1, ((TimestampedWindowStore <string, long>)store).All().ToList().Count);
            var it = ((TimestampedWindowStore <string, long>)store).Fetch("coucou", dt.AddSeconds(-5), dt.AddSeconds(5));

            Assert.IsTrue(it.MoveNext());
            Assert.AreEqual(1, it.Current.Value.Value.Value);
            driver.Dispose();
        }
Example #12
0
        private static void Main(string[] args)
        {
            var config = new StreamConfig <StringSerDes, StringSerDes>
            {
                ApplicationId = "test-test-driver-app"
            };

            StreamBuilder builder = new StreamBuilder();

            var table = builder
                        .Stream <string, string>("test")
                        .GroupByKey()
                        .WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromSeconds(10)))
                        .Count(InMemoryWindows <string, long> .As("count-store"));

            Topology t = builder.Build();

            using (var driver = new TopologyTestDriver(t, config))
            {
                DateTime dt         = DateTime.Now;
                var      inputTopic = driver.CreateInputTopic <string, string>("test");
                inputTopic.PipeInput("renault", "clio", dt);
                inputTopic.PipeInput("renault", "megane", dt.AddMilliseconds(10));
                Thread.Sleep((int)TimeSpan.FromSeconds(10).TotalMilliseconds);
                inputTopic.PipeInput("renault", "scenic", dt.AddSeconds(1));
                var store    = driver.GetWindowStore <string, long>("count-store");
                var elements = store.All().ToList();
            }
        }
Example #13
0
        public void TimeWindowingAggNothing()
        {
            var config = new StreamConfig <StringSerDes, StringSerDes>();

            config.ApplicationId = "test-window-stream";

            var builder = new StreamBuilder();

            builder
            .Stream <string, string>("topic")
            .GroupByKey()
            .WindowedBy(TumblingWindowOptions.Of(2000))
            .Aggregate(
                () => 0,
                (k, v, agg) => Math.Max(v.Length, agg),
                InMemoryWindows <string, int> .As("store").WithValueSerdes <Int32SerDes>())
            .ToStream()
            .To <StringTimeWindowedSerDes, Int32SerDes>("output");

            var topology = builder.Build();

            using (var driver = new TopologyTestDriver(topology, config))
            {
                var input    = driver.CreateInputTopic <string, string>("topic");
                var output   = driver.CreateOuputTopic("output", TimeSpan.FromSeconds(1), new StringTimeWindowedSerDes(), new Int64SerDes());
                var elements = output.ReadKeyValueList().ToList();
                Assert.AreEqual(0, elements.Count);
            }
        }
Example #14
0
        public void Windowing()
        {
            var builder = new StreamBuilder();

            builder.Stream <string, string>("topic")
            .GroupByKey()
            .WindowedBy(TumblingWindowOptions.Of(1000))
            .Count(RocksDbWindows <string, long> .As <StringSerDes, Int64SerDes>("rocksdb-w-store"));

            var config = new StreamConfig <StringSerDes, StringSerDes>();

            config.ApplicationId = "test-rocksdb-window-store";
            config.UseRandomRocksDbConfigForTest();

            Topology t = builder.Build();

            using (var driver = new TopologyTestDriver(t, config))
            {
                DateTime dt         = DateTime.Now;
                var      inputTopic = driver.CreateInputTopic <string, string>("topic");
                inputTopic.PipeInput("key1", "1", dt);
                inputTopic.PipeInput("key2", "2", dt);

                var store = driver.GetWindowStore <string, long>("rocksdb-w-store");
                Assert.IsNotNull(store);
                var k1 = store.FetchAll(dt.AddMinutes(-10), dt.AddMinutes(10)).ToList();

                Assert.AreEqual(2, k1.Count);
                Assert.AreEqual(1L, k1[0].Value);
                Assert.AreEqual(1L, k1[1].Value);
            }
            config.RemoveRocksDbFolderForTest();
        }
        public void WithNullMaterialize()
        {
            // CERTIFIED THAT SAME IF Materialize is null, a state store exist for count processor with a generated namestore
            var config = new StreamConfig <StringSerDes, StringSerDes>();
            var serdes = new StringSerDes();

            config.ApplicationId = "test-window-count";

            var builder = new StreamBuilder();
            Materialized <string, long, IWindowStore <Bytes, byte[]> > m = null;

            builder
            .Stream <string, string>("topic")
            .GroupByKey()
            .WindowedBy(TumblingWindowOptions.Of(2000))
            .Count(m);

            var    topology = builder.Build();
            TaskId id       = new TaskId {
                Id = 0, Partition = 0
            };
            var processorTopology = topology.Builder.BuildTopology(id);

            var supplier = new SyncKafkaSupplier();
            var producer = supplier.GetProducer(config.ToProducerConfig());
            var consumer = supplier.GetConsumer(config.ToConsumerConfig(), null);

            var        part = new TopicPartition("topic", 0);
            StreamTask task = new StreamTask(
                "thread-0",
                id,
                new List <TopicPartition> {
                part
            },
                processorTopology,
                consumer,
                config,
                supplier,
                null,
                new MockChangelogRegister(),
                new StreamMetricsRegistry());

            task.GroupMetadata = consumer as SyncConsumer;
            task.InitializeStateStores();
            task.InitializeTopology();
            task.RestorationIfNeeded();
            task.CompleteRestoration();

            Assert.AreEqual(1, task.Context.States.StateStoreNames.Count());
            var nameStore = task.Context.States.StateStoreNames.ElementAt(0);

            Assert.IsNotNull(nameStore);
            Assert.AreNotEqual(string.Empty, nameStore);
            var store = task.GetStore(nameStore);

            Assert.IsInstanceOf <ITimestampedWindowStore <string, long> >(store);
            Assert.AreEqual(0, (store as ITimestampedWindowStore <string, long>).All().ToList().Count);
        }
Example #16
0
        private static void Main(string[] args)
        {
            CancellationTokenSource source = new CancellationTokenSource();

            var config = new StreamConfig <StringSerDes, StringSerDes>();

            config.ApplicationId    = "test-app";
            config.BootstrapServers = "192.168.56.1:9092";
            config.SaslMechanism    = SaslMechanism.Plain;
            config.SaslUsername     = "******";
            config.SaslPassword     = "******";
            config.SecurityProtocol = SecurityProtocol.SaslPlaintext;
            config.AutoOffsetReset  = AutoOffsetReset.Earliest;
            config.NumStreamThreads = 1;

            StreamBuilder builder = new StreamBuilder();

            builder.Stream <string, string>("test")
            .GroupByKey()
            .WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromMinutes(1)))
            .Count(InMemoryWindows <string, long> .As("store"))
            .ToStream()
            .Map((k, v) => KeyValuePair.Create(k.Key, v.ToString()))
            .To("output");

            Topology t = builder.Build();
            //bool taskStart = false;

            KafkaStream stream = new KafkaStream(t, config);

            // Subscribe state changed
            //stream.StateChanged += (old, @new) =>
            //{
            //    if (!taskStart && @new == KafkaStream.State.RUNNING) // If new state is running, we can quering state store.
            //    {
            //        Task.Factory.StartNew(() =>
            //        {
            //            while (!source.Token.IsCancellationRequested)
            //            {
            //                var store = stream.Store(StoreQueryParameters.FromNameAndType("store", QueryableStoreTypes.WindowStore<string, long>()));
            //                var items = store.All().ToList();
            //                Thread.Sleep(500);
            //            }
            //        }, source.Token);
            //        taskStart = true;
            //    }
            //};

            Console.CancelKeyPress += (o, e) =>
            {
                source.Cancel();
                stream.Close();
            };

            stream.Start(source.Token);
        }
Example #17
0
 private void BuildTopology(StreamBuilder builder)
 {
     builder.Stream <string, ObjectA, StringSerDes, JSONSerDes <ObjectA> >("source")
     .Map((key, value) => new KeyValuePair <string, ObjectA>(value.Symbol, value))
     .GroupByKey <StringSerDes, JSONSerDes <ObjectA> >()
     .WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromMinutes(5)))
     .Aggregate <ObjectB, JSONSerDes <ObjectB> >(
         () => new ObjectB(),
         (key, ObjectA, ObjectB) => ObjectBHelper.CreateObjectB(key, ObjectA, ObjectB))
     .ToStream()
     .Map((key, ObjectB) => new KeyValuePair <string, ObjectB>(key.Key, ObjectB))
     .To <StringSerDes, JSONSerDes <ObjectB> >("sink");
 }
Example #18
0
        public async Task GetWindowStateStore()
        {
            var timeout = TimeSpan.FromSeconds(10);

            bool     isRunningState = false;
            DateTime dt             = DateTime.Now;

            var config = new StreamConfig <StringSerDes, StringSerDes>();

            config.ApplicationId    = "test";
            config.BootstrapServers = "127.0.0.1";

            var builder = new StreamBuilder();

            builder
            .Stream <string, string>("test")
            .GroupByKey()
            .WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromMinutes(1)))
            .Count(InMemoryWindows <string, long> .As("store"));

            var t      = builder.Build();
            var stream = new KafkaStream(t, config, new SyncKafkaSupplier());

            stream.StateChanged += (old, @new) =>
            {
                if (@new.Equals(KafkaStream.State.RUNNING))
                {
                    isRunningState = true;
                }
            };
            await stream.StartAsync();

            while (!isRunningState)
            {
                Thread.Sleep(250);
                if (DateTime.Now > dt + timeout)
                {
                    break;
                }
            }
            Assert.IsTrue(isRunningState);

            if (isRunningState)
            {
                var store = stream.Store(StoreQueryParameters.FromNameAndType("store", QueryableStoreTypes.WindowStore <string, long>()));
                Assert.IsNotNull(store);
            }

            stream.Dispose();
        }
Example #19
0
        public void GetWStateStoreInvalidStateStoreException()
        {
            var      timeout = TimeSpan.FromSeconds(10);
            var      source  = new CancellationTokenSource();
            bool     state   = false;
            DateTime dt      = DateTime.Now;

            var config = new StreamConfig <StringSerDes, StringSerDes>();

            config.ApplicationId    = "test";
            config.BootstrapServers = "127.0.0.1";

            var builder = new StreamBuilder();

            builder
            .Stream <string, string>("test")
            .GroupByKey()
            .WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromMinutes(1)))
            .Count(InMemoryWindows <string, long> .As("store"));

            var t      = builder.Build();
            var stream = new KafkaStream(t, config, new SyncKafkaSupplier());

            stream.StateChanged += (old, @new) =>
            {
                if ([email protected](KafkaStream.State.RUNNING))
                {
                    if (!state)
                    {
                        Assert.Throws <InvalidStateStoreException>(() => stream.Store(StoreQueryParameters.FromNameAndType("store", QueryableStoreTypes.WindowStore <string, long>())));
                        state = true;
                    }
                }
            };
            stream.Start(source.Token);
            while (!state)
            {
                Thread.Sleep(250);
                if (DateTime.Now > dt + timeout)
                {
                    break;
                }
            }
            Assert.IsTrue(state);

            source.Cancel();
            stream.Close();
        }
Example #20
0
        public void WithNullMaterialize()
        {
            // CERTIFIED THAT SAME IF Materialize is null, a state store exist for count processor with a generated namestore
            var config = new StreamConfig <StringSerDes, StringSerDes>();
            var serdes = new StringSerDes();

            config.ApplicationId = "test-window-count";

            var builder = new StreamBuilder();
            Materialized <string, int, IWindowStore <Bytes, byte[]> > m = null;

            builder
            .Stream <string, string>("topic")
            .GroupByKey()
            .WindowedBy(TumblingWindowOptions.Of(2000))
            .Aggregate(
                () => 0,
                (k, v, agg) => Math.Max(v.Length, agg),
                m);

            var    topology = builder.Build();
            TaskId id       = new TaskId {
                Id = 0, Partition = 0
            };
            var processorTopology = topology.Builder.BuildTopology(id);

            var supplier = new SyncKafkaSupplier();
            var producer = supplier.GetProducer(config.ToProducerConfig());
            var consumer = supplier.GetConsumer(config.ToConsumerConfig(), null);


            var        part = new TopicPartition("topic", 0);
            StreamTask task = new StreamTask(
                "thread-0",
                id,
                new List <TopicPartition> {
                part
            },
                processorTopology,
                consumer,
                config,
                supplier,
                null);

            task.GroupMetadata = consumer as SyncConsumer;
            Assert.Throws <StreamsException>(() => task.InitializeStateStores());
        }
Example #21
0
        public void WithNullMaterialize()
        {
            // CERTIFIED THAT SAME IF Materialize is null, a state store exist for count processor with a generated namestore
            var config = new StreamConfig();
            var serdes = new StringSerDes();

            config.ApplicationId = "test-window-reduce";

            var builder = new StreamBuilder();

            builder
            .Stream <string, string>("topic")
            .GroupByKey()
            .WindowedBy(TumblingWindowOptions.Of(2000))
            .Reduce((v1, v2) => v1.Length > v2.Length ? v1 : v2);

            var    topology = builder.Build();
            TaskId id       = new TaskId {
                Id = 0, Partition = 0
            };
            var processorTopology = topology.Builder.BuildTopology(id);

            var supplier = new SyncKafkaSupplier();
            var producer = supplier.GetProducer(config.ToProducerConfig());
            var consumer = supplier.GetConsumer(config.ToConsumerConfig(), null);

            var        part = new TopicPartition("topic", 0);
            StreamTask task = new StreamTask(
                "thread-0",
                id,
                new List <TopicPartition> {
                part
            },
                processorTopology,
                consumer,
                config,
                supplier,
                null,
                new MockChangelogRegister(),
                new StreamMetricsRegistry());

            task.GroupMetadata = consumer as SyncConsumer;
            Assert.Throws <StreamsException>(() => task.InitializeStateStores());
        }
Example #22
0
        public void TimeWindowingAggWithMaterialize()
        {
            var config = new StreamConfig <StringSerDes, StringSerDes>();

            config.ApplicationId = "test-window-stream";

            var builder = new StreamBuilder();

            builder
            .Stream <string, string>("topic")
            .GroupByKey()
            .WindowedBy(TumblingWindowOptions.Of(2000))
            .Aggregate(
                () => 0,
                (k, v, agg) => Math.Max(v.Length, agg),
                InMemoryWindows <string, int> .As("store").WithValueSerdes <Int32SerDes>())
            .ToStream()
            .To <StringTimeWindowedSerDes, Int32SerDes>("output");

            var topology = builder.Build();

            using (var driver = new TopologyTestDriver(topology, config))
            {
                var input  = driver.CreateInputTopic <string, string>("topic");
                var output = driver.CreateOuputTopic("output", TimeSpan.FromSeconds(1), new StringTimeWindowedSerDes(), new Int32SerDes());
                input.PipeInput("test", "1");
                input.PipeInput("test", "230");
                input.PipeInput("test", "32");
                var elements = output.ReadKeyValueList().ToList();
                Assert.AreEqual(3, elements.Count);
                Assert.AreEqual("test", elements[0].Message.Key.Key);
                Assert.AreEqual((long)TimeSpan.FromSeconds(10).TotalMilliseconds, elements[0].Message.Key.Window.EndMs - elements[0].Message.Key.Window.StartMs);
                Assert.AreEqual(1, elements[0].Message.Value);
                Assert.AreEqual("test", elements[1].Message.Key.Key);
                Assert.AreEqual(elements[0].Message.Key.Window, elements[1].Message.Key.Window);
                Assert.AreEqual(3, elements[1].Message.Value);
                Assert.AreEqual("test", elements[2].Message.Key.Key);
                Assert.AreEqual(elements[0].Message.Key.Window, elements[2].Message.Key.Window);
                Assert.AreEqual(3, elements[2].Message.Value);
            }
        }
Example #23
0
        public void WindowStateStoreNotExist()
        {
            var config = new StreamConfig <StringSerDes, StringSerDes>();

            config.ApplicationId = "test-config";

            var builder = new StreamBuilder();

            builder.Stream <string, string>("test")
            .GroupByKey()
            .WindowedBy(TumblingWindowOptions.Of(2000))
            .Count(InMemoryWindows <string, long> .As("store"));

            var driver = new TaskSynchronousTopologyDriver("client", builder.Build().Builder, config, config, default);

            driver.CreateInputTopic("test", new StringSerDes(), new StringSerDes());
            var store = driver.GetStateStore <string, string>("store2");

            Assert.IsNull(store);
            driver.Dispose();
        }
Example #24
0
        public void WithNullValueSerDes()
        {
            // WITH VALUE NULL SERDES, in running KeySerdes must be StringSerdes, and ValueSerdes Int64SerDes
            var config = new StreamConfig <StringSerDes, StringSerDes>();

            config.ApplicationId = "test-window-count";

            var builder = new StreamBuilder();

            Materialized <string, long, WindowStore <Bytes, byte[]> > m =
                Materialized <string, long, WindowStore <Bytes, byte[]> >
                .Create("count-store")
                .With(null, null);

            builder
            .Stream <string, string>("topic")
            .GroupByKey()
            .WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromSeconds(5)))
            .Count(m)
            .ToStream()
            .To <StringTimeWindowedSerDes, Int64SerDes>("output-topic");

            var topology = builder.Build();

            using (var driver = new TopologyTestDriver(topology, config))
            {
                var input  = driver.CreateInputTopic <string, string>("topic");
                var output = driver.CreateOuputTopic("output-topic", TimeSpan.FromSeconds(1), new StringTimeWindowedSerDes(), new Int64SerDes());
                input.PipeInput("test", "1");
                input.PipeInput("test-test", "30");
                var records = output.ReadKeyValueList().ToList();
                Assert.AreEqual(2, records.Count);
                Assert.AreEqual("test", records[0].Message.Key.Key);
                Assert.AreEqual(1, records[0].Message.Value);
                Assert.AreEqual("test-test", records[1].Message.Key.Key);
                Assert.AreEqual(1, records[1].Message.Value);
                Assert.AreEqual(records[0].Message.Key.Window, records[1].Message.Key.Window);
            }
        }
Example #25
0
        public void TimeWindowingCountWithMaterialize()
        {
            var config = new StreamConfig <StringSerDes, StringSerDes>();

            config.ApplicationId = "test-window-stream";

            var builder = new StreamBuilder();

            builder
            .Stream <string, string>("topic")
            .GroupByKey()
            .WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromSeconds(10)))
            .Count(Materialized <string, long, WindowStore <Bytes, byte[]> > .Create("count-store"))
            .ToStream()
            .To <StringTimeWindowedSerDes, Int64SerDes>("output");

            var topology = builder.Build();

            using (var driver = new TopologyTestDriver(topology, config))
            {
                var input  = driver.CreateInputTopic <string, string>("topic");
                var output = driver.CreateOuputTopic("output", TimeSpan.FromSeconds(1), new StringTimeWindowedSerDes(), new Int64SerDes());
                input.PipeInput("test", "1");
                input.PipeInput("test", "2");
                input.PipeInput("test", "3");
                var elements = output.ReadKeyValueList().ToList();
                Assert.AreEqual(3, elements.Count);
                Assert.AreEqual("test", elements[0].Message.Key.Key);
                Assert.AreEqual((long)TimeSpan.FromSeconds(10).TotalMilliseconds, elements[0].Message.Key.Window.EndMs - elements[0].Message.Key.Window.StartMs);
                Assert.AreEqual(1, elements[0].Message.Value);
                Assert.AreEqual("test", elements[1].Message.Key.Key);
                Assert.AreEqual(elements[0].Message.Key.Window, elements[1].Message.Key.Window);
                Assert.AreEqual(2, elements[1].Message.Value);
                Assert.AreEqual("test", elements[2].Message.Key.Key);
                Assert.AreEqual(elements[0].Message.Key.Window, elements[2].Message.Key.Window);
                Assert.AreEqual(3, elements[2].Message.Value);
            }
        }
        public void TimeWindowingReduce()
        {
            var config = new StreamConfig <StringSerDes, StringSerDes>();

            config.ApplicationId = "test-window-stream";

            var builder = new StreamBuilder();

            builder
            .Stream <string, string>("topic")
            .GroupByKey()
            .WindowedBy(TumblingWindowOptions.Of(2000))
            .Reduce((v1, v2) => v1.Length > v2.Length ? v1 : v2)
            .ToStream()
            .To <StringTimeWindowedSerDes, StringSerDes>("output");

            var topology = builder.Build();

            using (var driver = new TopologyTestDriver(topology, config))
            {
                var input  = driver.CreateInputTopic <string, string>("topic");
                var output = driver.CreateOuputTopic("output", TimeSpan.FromSeconds(1), new StringTimeWindowedSerDes(), new StringSerDes());
                input.PipeInput("test", "1");
                input.PipeInput("test", "230");
                input.PipeInput("test", "32");
                var elements = output.ReadKeyValueList().ToList();
                Assert.AreEqual(3, elements.Count);
                Assert.AreEqual("test", elements[0].Message.Key.Key);
                Assert.AreEqual((long)TimeSpan.FromSeconds(10).TotalMilliseconds, elements[0].Message.Key.Window.EndMs - elements[0].Message.Key.Window.StartMs);
                Assert.AreEqual("1", elements[0].Message.Value);
                Assert.AreEqual("test", elements[1].Message.Key.Key);
                Assert.AreEqual(elements[0].Message.Key.Window, elements[1].Message.Key.Window);
                Assert.AreEqual("230", elements[1].Message.Value);
                Assert.AreEqual("test", elements[2].Message.Key.Key);
                Assert.AreEqual(elements[0].Message.Key.Window, elements[2].Message.Key.Window);
                Assert.AreEqual("230", elements[2].Message.Value);
            }
        }
Example #27
0
        public void StoresPersistentKeyValueStoreTest()
        {
            var builder = new StreamBuilder();

            builder.Stream <string, string>("topic")
            .GroupByKey()
            .WindowedBy(TumblingWindowOptions.Of(1000))
            .Count(
                Materialized <string, long, IWindowStore <Bytes, byte[]> > .Create(
                    Streamiz.Kafka.Net.State.Stores.PersistentWindowStore(
                        "rocksdb-w-store",
                        TimeSpan.FromDays(1),
                        TimeSpan.FromSeconds(1))));

            var config = new StreamConfig <StringSerDes, StringSerDes>();

            config.ApplicationId = "test-rocksdb-window-store";
            config.UseRandomRocksDbConfigForTest();

            Topology t = builder.Build();

            using (var driver = new TopologyTestDriver(t, config))
            {
                DateTime dt         = DateTime.Now;
                var      inputTopic = driver.CreateInputTopic <string, string>("topic");
                inputTopic.PipeInput("abc", "1", dt);
                inputTopic.PipeInput("key1", "1", dt);
                inputTopic.PipeInput("test", "1", dt);

                var store = driver.GetWindowStore <string, long>("rocksdb-w-store");
                Assert.IsNotNull(store);
                var k1 = store.FetchAll(dt.AddMinutes(-10), dt.AddMinutes(10)).ToList();

                Assert.AreEqual(3, k1.Count);
            }
            config.RemoveRocksDbFolderForTest();
        }
Example #28
0
        static async System.Threading.Tasks.Task Main(string[] args)
        {
            var config = new StreamConfig <StringSerDes, StringSerDes>();

            config.ApplicationId    = "test-app";
            config.BootstrapServers = "localhost:9093";


            StreamBuilder builder = new StreamBuilder();

            builder.Stream <string, string>("evenements")
            .GroupByKey()
            .WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromMinutes(1)))
            .Aggregate(() => "", (k, v, va) => va += v)
            .ToStream()
            .Print(Printed <Windowed <String>, String> .ToOut());

            Topology    t      = builder.Build();
            KafkaStream stream = new KafkaStream(t, config);

            Console.CancelKeyPress += (o, e) => stream.Dispose();

            await stream.StartAsync();
        }
Example #29
0
        public async Task GetWindowElementInStateStore()
        {
            var timeout = TimeSpan.FromSeconds(10);

            bool     isRunningState = false;
            DateTime dt             = DateTime.Now;

            var config = new StreamConfig <StringSerDes, StringSerDes>();

            config.ApplicationId    = "test";
            config.BootstrapServers = "127.0.0.1";
            config.PollMs           = 10;

            var supplier = new SyncKafkaSupplier();
            var producer = supplier.GetProducer(config.ToProducerConfig());

            var builder = new StreamBuilder();

            builder
            .Stream <string, string>("test")
            .GroupByKey()
            .WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromMinutes(1)))
            .Count(InMemoryWindows <string, long> .As("store"));

            var t      = builder.Build();
            var stream = new KafkaStream(t, config, supplier);

            stream.StateChanged += (old, @new) =>
            {
                if (@new.Equals(KafkaStream.State.RUNNING))
                {
                    isRunningState = true;
                }
            };
            await stream.StartAsync();

            while (!isRunningState)
            {
                Thread.Sleep(250);
                if (DateTime.Now > dt + timeout)
                {
                    break;
                }
            }
            Assert.IsTrue(isRunningState);

            if (isRunningState)
            {
                var serdes = new StringSerDes();
                dt = DateTime.Now;
                producer.Produce("test",
                                 new Confluent.Kafka.Message <byte[], byte[]>
                {
                    Key       = serdes.Serialize("key1", new SerializationContext()),
                    Value     = serdes.Serialize("coucou", new SerializationContext()),
                    Timestamp = new Confluent.Kafka.Timestamp(dt)
                });
                Thread.Sleep(50);
                var store = stream.Store(StoreQueryParameters.FromNameAndType("store", QueryableStoreTypes.WindowStore <string, long>()));
                Assert.IsNotNull(store);
                var @enum = store.All();
                Assert.AreEqual(1, store.All().ToList().Count);
                var item = store.Fetch("key1", dt.AddMinutes(-1), dt.AddMinutes(1));
                Assert.IsNotNull(item);
                Assert.IsTrue(item.MoveNext());
                Assert.IsTrue(item.Current.HasValue);
                Assert.AreEqual(1, item.Current.Value.Value);
                item.Dispose();
            }

            stream.Dispose();
        }