public void TestConsume() { var consumerConfig = new ConsumerConfig(); consumerConfig.GroupId = "cg"; consumerConfig.ClientId = "cg-0"; var supplier = new MockKafkaSupplier(2); var producer = supplier.GetProducer(new ProducerConfig()); producer.Produce("topic", new Message <byte[], byte[]> { Key = new byte[1] { 42 }, Value = new byte[1] { 12 } }); var c1 = supplier.GetConsumer(consumerConfig, null); c1.Subscribe(new List <string> { "topic" }); var item1 = c1.Consume(); Assert.IsNotNull(item1); var item2 = c1.Consume(); Assert.IsNull(item2); }
public void TestAssignment() { var consumerConfig = new ConsumerConfig(); var consumerConfig2 = new ConsumerConfig(); consumerConfig.GroupId = "cg"; consumerConfig2.GroupId = "cg"; consumerConfig.ClientId = "cg-0"; consumerConfig2.ClientId = "cg-1"; var supplier = new MockKafkaSupplier(2); var c1 = supplier.GetConsumer(consumerConfig, null); var c2 = supplier.GetConsumer(consumerConfig2, null); c1.Subscribe(new List <string> { "topic1", "topic2" }); c2.Subscribe(new List <string> { "topic1", "topic2" }); c1.Consume(); Assert.AreEqual(2, c1.Assignment.Count); Assert.AreEqual(2, c2.Assignment.Count); }
public void TestConsumeRecords2() { var consumerConfig = new ConsumerConfig(); consumerConfig.GroupId = "cg"; consumerConfig.ClientId = "cg-0"; var supplier = new MockKafkaSupplier(2); var producer = supplier.GetProducer(new ProducerConfig()); producer.Produce("topic", new Message <byte[], byte[]> { Key = new byte[1] { 42 }, Value = new byte[1] { 12 } }); var c1 = supplier.GetConsumer(consumerConfig, null); c1.Subscribe(new List <string> { "topic" }); var item = c1.ConsumeRecords(TimeSpan.FromSeconds(1)).ToList(); Assert.IsNotNull(item); Assert.AreEqual(1, item.Count); }
public void Init() { config.ApplicationId = "test-stream-thread"; config.StateDir = Guid.NewGuid().ToString(); config.Guarantee = ProcessingGuarantee.AT_LEAST_ONCE; config.PollMs = 10; mockKafkaSupplier = new MockKafkaSupplier(4); var builder = new StreamBuilder(); builder.Stream <string, string>("topic").To("topic2"); var topo = builder.Build(); thread1 = StreamThread.Create( "thread-0", "c0", topo.Builder, new StreamMetricsRegistry(), config, mockKafkaSupplier, mockKafkaSupplier.GetAdmin(config.ToAdminConfig("admin")), 0) as StreamThread; thread2 = StreamThread.Create( "thread-1", "c1", topo.Builder, new StreamMetricsRegistry(), config, mockKafkaSupplier, mockKafkaSupplier.GetAdmin(config.ToAdminConfig("admin")), 1) as StreamThread; }
public void Initialize() { streamMetricsRegistry = new StreamMetricsRegistry(Guid.NewGuid().ToString(), MetricsRecordingLevel.INFO); config.ApplicationId = "test-stream-thread"; config.StateDir = Guid.NewGuid().ToString(); config.Guarantee = ProcessingGuarantee.AT_LEAST_ONCE; config.PollMs = 10; config.MaxPollRecords = 10; config.CommitIntervalMs = 10; config.MetricsRecording = MetricsRecordingLevel.INFO; mockKafkaSupplier = new MockKafkaSupplier(numberPartitions); var builder = new StreamBuilder(); builder.Stream <string, string>("topic").To("topic2"); var topo = builder.Build(); thread = StreamThread.Create( threadId, "c1", topo.Builder, streamMetricsRegistry, config, mockKafkaSupplier, mockKafkaSupplier.GetAdmin(config.ToAdminConfig("admin")), 1) as StreamThread; }
public void TestPauseResumeAssign() { var consumerConfig = new ConsumerConfig(); consumerConfig.GroupId = "cg"; consumerConfig.ClientId = "cg-0"; var supplier = new MockKafkaSupplier(1); var producer = supplier.GetProducer(new ProducerConfig()); producer.Produce(new TopicPartition("topic", 0), new Message <byte[], byte[]> { Key = new byte[1] { 1 }, Value = new byte[1] { 1 } }); producer.Produce(new TopicPartition("topic", 0), new Message <byte[], byte[]> { Key = new byte[1] { 2 }, Value = new byte[1] { 2 } }); var c1 = supplier.GetConsumer(consumerConfig, null); c1.Assign(new TopicPartition("topic", 0).ToSingle()); var r1 = c1.Consume(); var r2 = c1.Consume(); var r3 = c1.Consume(); Assert.IsNotNull(r1); Assert.IsNotNull(r2); Assert.IsNull(r3); c1.Pause(c1.Assignment); c1.Assign(new List <TopicPartition>()); c1.Assign(new TopicPartition("topic", 0).ToSingle()); producer.Produce(new TopicPartition("topic", 0), new Message <byte[], byte[]> { Key = new byte[1] { 3 }, Value = new byte[1] { 3 } }); c1.Resume(new TopicPartition("topic", 0).ToSingle()); var r4 = c1.Consume(); Assert.IsNotNull(r4); }
public void RepartitionTestJoinTopology() { var config = new StreamConfig <StringSerDes, StringSerDes> { ApplicationId = "test-repartition-processor" }; var supplier = new MockKafkaSupplier(10); StreamBuilder builder = new StreamBuilder(); IKStream <string, string> stream1 = builder .Stream <string, string>("topic") .Map((k, v) => KeyValuePair.Create(k.ToUpper(), v)); IKStream <string, string> stream2 = builder.Stream <string, string>("topic2"); stream1.Join(stream2, (v1, v2) => $"{v1}-{v2}", JoinWindowOptions.Of(TimeSpan.FromMinutes(1)), StreamJoinProps.As <string, string, string>("join-store")) .To("output"); Topology t = builder.Build(); using (var driver = new TopologyTestDriver(t.Builder, config, TopologyTestDriver.Mode.ASYNC_CLUSTER_IN_MEMORY, supplier)) { var inputTopic = driver.CreateInputTopic <string, string>("topic"); var inputTopic2 = driver.CreateInputTopic <string, string>("topic2"); var outputTopic = driver.CreateOuputTopic <string, string>("output"); inputTopic.PipeInput("test", "coucou"); inputTopic2.PipeInput("TEST", "sylvain"); inputTopic2.PipeInput("TEST2", "antoine"); inputTopic.PipeInput("test2", "test"); var records = IntegrationTestUtils .WaitUntilMinKeyValueRecordsReceived(outputTopic, 2) .ToUpdateDictionary(r => r.Message.Key, r => r.Message.Value); Assert.IsNotNull(records); Assert.AreEqual(2, records.Count); Assert.AreEqual("coucou-sylvain", records["TEST"]); Assert.AreEqual("test-antoine", records["TEST2"]); } }
public void RepartitionInternalTopicOnCascace() { var config = new StreamConfig <StringSerDes, StringSerDes> { ApplicationId = "test-repartition-processor-on-cascade" }; StreamBuilder builder = new StreamBuilder(); var table = builder.Table <string, string>("input-table"); builder .Stream <string, string>("topic") .SelectKey((k, v) => v.ToUpper()) .Join(table, (v, t) => $"{v}:{t}") .To("output"); Topology t = builder.Build(); MockKafkaSupplier supplier = new MockKafkaSupplier(4); using (var driver = new TopologyTestDriver(t.Builder, config, TopologyTestDriver.Mode.ASYNC_CLUSTER_IN_MEMORY, supplier)) { var inputTopic = driver.CreateInputTopic <string, string>("topic"); var inputTableTopic = driver.CreateInputTopic <string, string>("input-table"); var outputTopic = driver.CreateOuputTopic <string, string>("output"); inputTableTopic.PipeInput("PRODUCT1", "P1"); inputTableTopic.PipeInput("PRODUCT2", "P2"); inputTopic.PipeInput("test", "product1"); inputTopic.PipeInput("test", "product2"); var records = IntegrationTestUtils .WaitUntilMinKeyValueRecordsReceived(outputTopic, 2) .ToUpdateDictionary(r => r.Message.Key, r => r.Message.Value); Assert.IsNotNull(records); Assert.AreEqual(2, records.Count); Assert.AreEqual("product2:P2", records["PRODUCT2"]); Assert.AreEqual("product1:P1", records["PRODUCT1"]); } }
public void Init() { token1 = new System.Threading.CancellationTokenSource(); token2 = new System.Threading.CancellationTokenSource(); config.ApplicationId = "test-stream-thread"; config.StateDir = Path.Combine(".", Guid.NewGuid().ToString()); config.Guarantee = ProcessingGuarantee.AT_LEAST_ONCE; config.PollMs = 10; mockKafkaSupplier = new MockKafkaSupplier(2, 0); var builder = new StreamBuilder(); builder.Table("topic", InMemory <string, string> .As("store").WithLoggingEnabled()); var topo = builder.Build(); topo.Builder.RewriteTopology(config); topo.Builder.BuildTopology(); thread1 = StreamThread.Create( "thread-0", "c0", topo.Builder, new StreamMetricsRegistry(), config, mockKafkaSupplier, mockKafkaSupplier.GetAdmin(config.ToAdminConfig("admin")), 0) as StreamThread; thread2 = StreamThread.Create( "thread-1", "c1", topo.Builder, new StreamMetricsRegistry(), config, mockKafkaSupplier, mockKafkaSupplier.GetAdmin(config.ToAdminConfig("admin")), 1) as StreamThread; var internalTopicManager = new DefaultTopicManager(config, mockKafkaSupplier.GetAdmin(config.ToAdminConfig("admin"))); InternalTopicManagerUtils .New() .CreateInternalTopicsAsync(internalTopicManager, topo.Builder).GetAwaiter(); }
public void RepartitionTestTopologyAsyncHighNumberPartititon() { var config = new StreamConfig <StringSerDes, StringSerDes> { ApplicationId = "test-repartition-processor" }; var supplier = new MockKafkaSupplier(30); StreamBuilder builder = new StreamBuilder(); builder .Stream <string, string>("topic") .Map((k, v) => KeyValuePair.Create(k.ToUpper(), v)) .GroupByKey() .Count() .ToStream() .To("output"); Topology t = builder.Build(); using (var driver = new TopologyTestDriver(t.Builder, config, TopologyTestDriver.Mode.ASYNC_CLUSTER_IN_MEMORY, supplier)) { var inputTopic = driver.CreateInputTopic <string, string>("topic"); var outputTopic = driver.CreateOuputTopic <string, long, StringSerDes, Int64SerDes>("output"); inputTopic.PipeInput("test", "test1"); inputTopic.PipeInput("test", "test2"); inputTopic.PipeInput("test", "test3"); inputTopic.PipeInput("test", "test4"); inputTopic.PipeInput("test", "test5"); var records = IntegrationTestUtils .WaitUntilMinKeyValueRecordsReceived(outputTopic, 5) .ToUpdateDictionary(r => r.Message.Key, r => r.Message.Value); Assert.IsNotNull(records); Assert.AreEqual(1, records.Count); Assert.AreEqual(5, records["TEST"]); } }
public void StreamThreadNormalWorkflowWithRebalancing() { List <ThreadState> allStates = new List <ThreadState>(); var expectedStates = new List <ThreadState> { ThreadState.CREATED, ThreadState.STARTING, ThreadState.PARTITIONS_ASSIGNED, ThreadState.RUNNING, ThreadState.PARTITIONS_REVOKED, ThreadState.PARTITIONS_ASSIGNED, ThreadState.RUNNING, ThreadState.PENDING_SHUTDOWN, ThreadState.DEAD }; var source = new System.Threading.CancellationTokenSource(); var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test"; config.Guarantee = ProcessingGuarantee.AT_LEAST_ONCE; config.PollMs = 1; var consumeConfig = config.Clone(); consumeConfig.ApplicationId = "consume-test"; var serdes = new StringSerDes(); var builder = new StreamBuilder(); builder.Stream <string, string>("topic").To("topic2"); var topo = builder.Build(); var supplier = new MockKafkaSupplier(4); var producer = supplier.GetProducer(consumeConfig.ToProducerConfig()); var consumer = supplier.GetConsumer(consumeConfig.ToConsumerConfig("test-consum"), null); consumer.Subscribe("topic2"); var thread = StreamThread.Create( "thread-0", "c0", topo.Builder, config, supplier, supplier.GetAdmin(config.ToAdminConfig("admin")), 0) as StreamThread; allStates.Add(thread.State); thread.StateChanged += (t, o, n) => { Assert.IsInstanceOf <ThreadState>(n); allStates.Add(n as ThreadState); }; thread.Start(source.Token); // WAIT PARTITONS ASSIGNED System.Threading.Thread.Sleep(50); var thread2 = StreamThread.Create( "thread-1", "c1", topo.Builder, config, supplier, supplier.GetAdmin(config.ToAdminConfig("admin")), 1) as StreamThread; thread2.Start(source.Token); // WAIT PARTITONS REBALANCING System.Threading.Thread.Sleep(50); producer.Produce("topic", new Message <byte[], byte[]> { Key = serdes.Serialize("key1", new SerializationContext()), Value = serdes.Serialize("coucou", new SerializationContext()) }); //WAIT STREAMTHREAD PROCESS MESSAGE System.Threading.Thread.Sleep(100); var message = consumer.Consume(100); // 2 CONSUMER FOR THE SAME GROUP ID => TOPIC WITH 4 PARTITIONS Assert.AreEqual(2, thread.ActiveTasks.Count()); Assert.AreEqual(2, thread2.ActiveTasks.Count()); source.Cancel(); thread.Dispose(); thread2.Dispose(); Assert.AreEqual("key1", serdes.Deserialize(message.Message.Key, new SerializationContext())); Assert.AreEqual("coucou", serdes.Deserialize(message.Message.Value, new SerializationContext())); Assert.AreEqual(expectedStates, allStates); // Destroy in memory cluster supplier.Destroy(); }
public void TestConsumeSaveOffset() { var consumerConfig = new ConsumerConfig(); consumerConfig.GroupId = "cg"; consumerConfig.ClientId = "cg-0"; var consumerConfig1 = new ConsumerConfig(); consumerConfig1.GroupId = "cg"; consumerConfig1.ClientId = "cg-1"; var supplier = new MockKafkaSupplier(2); var producer = supplier.GetProducer(new ProducerConfig()); producer.Produce(new TopicPartition("topic", 0), new Message <byte[], byte[]> { Key = new byte[1] { 42 }, Value = new byte[1] { 12 } }); producer.Produce(new TopicPartition("topic", 1), new Message <byte[], byte[]> { Key = new byte[1] { 43 }, Value = new byte[1] { 13 } }); var c1 = supplier.GetConsumer(consumerConfig, null); var c2 = supplier.GetConsumer(consumerConfig1, null); c1.Subscribe(new List <string> { "topic" }); c1.Consume(); c2.Subscribe(new List <string> { "topic" }); c2.Consume(); c1.Consume(); c2.Consume(); Assert.AreEqual(1, c1.Assignment.Count); Assert.AreEqual(1, c2.Assignment.Count); c1.Unsubscribe(); c2.Unsubscribe(); c1.Subscribe(new List <string> { "topic" }); c1.Consume(); c2.Subscribe(new List <string> { "topic" }); c2.Consume(); c2.Consume(); c1.Consume(); c2.Consume(); Assert.AreEqual(1, c1.Assignment.Count); Assert.AreEqual(1, c2.Assignment.Count); }