public void TestConsume()
        {
            var consumerConfig = new ConsumerConfig();

            consumerConfig.GroupId  = "cg";
            consumerConfig.ClientId = "cg-0";

            var supplier = new MockKafkaSupplier(2);
            var producer = supplier.GetProducer(new ProducerConfig());

            producer.Produce("topic", new Message <byte[], byte[]> {
                Key = new byte[1] {
                    42
                }, Value = new byte[1] {
                    12
                }
            });

            var c1 = supplier.GetConsumer(consumerConfig, null);

            c1.Subscribe(new List <string> {
                "topic"
            });

            var item1 = c1.Consume();

            Assert.IsNotNull(item1);
            var item2 = c1.Consume();

            Assert.IsNull(item2);
        }
        public void TestConsumeRecords2()
        {
            var consumerConfig = new ConsumerConfig();

            consumerConfig.GroupId  = "cg";
            consumerConfig.ClientId = "cg-0";

            var supplier = new MockKafkaSupplier(2);
            var producer = supplier.GetProducer(new ProducerConfig());

            producer.Produce("topic", new Message <byte[], byte[]> {
                Key = new byte[1] {
                    42
                }, Value = new byte[1] {
                    12
                }
            });

            var c1 = supplier.GetConsumer(consumerConfig, null);

            c1.Subscribe(new List <string> {
                "topic"
            });

            var item = c1.ConsumeRecords(TimeSpan.FromSeconds(1)).ToList();

            Assert.IsNotNull(item);
            Assert.AreEqual(1, item.Count);
        }
Beispiel #3
0
        public void TestPauseResumeAssign()
        {
            var consumerConfig = new ConsumerConfig();

            consumerConfig.GroupId  = "cg";
            consumerConfig.ClientId = "cg-0";


            var supplier = new MockKafkaSupplier(1);
            var producer = supplier.GetProducer(new ProducerConfig());

            producer.Produce(new TopicPartition("topic", 0), new Message <byte[], byte[]> {
                Key = new byte[1] {
                    1
                }, Value = new byte[1] {
                    1
                }
            });
            producer.Produce(new TopicPartition("topic", 0), new Message <byte[], byte[]> {
                Key = new byte[1] {
                    2
                }, Value = new byte[1] {
                    2
                }
            });

            var c1 = supplier.GetConsumer(consumerConfig, null);

            c1.Assign(new TopicPartition("topic", 0).ToSingle());
            var r1 = c1.Consume();
            var r2 = c1.Consume();
            var r3 = c1.Consume();

            Assert.IsNotNull(r1);
            Assert.IsNotNull(r2);
            Assert.IsNull(r3);

            c1.Pause(c1.Assignment);

            c1.Assign(new List <TopicPartition>());

            c1.Assign(new TopicPartition("topic", 0).ToSingle());

            producer.Produce(new TopicPartition("topic", 0), new Message <byte[], byte[]> {
                Key = new byte[1] {
                    3
                }, Value = new byte[1] {
                    3
                }
            });

            c1.Resume(new TopicPartition("topic", 0).ToSingle());

            var r4 = c1.Consume();

            Assert.IsNotNull(r4);
        }
Beispiel #4
0
        public void StreamThreadNormalWorkflowWithRebalancing()
        {
            var consumeConfig = config.Clone();

            consumeConfig.ApplicationId = "consume-test";

            var serdes  = new StringSerDes();
            var builder = new StreamBuilder();

            builder.Stream <string, string>("topic").To("topic2");

            var topo = builder.Build();

            var producer = mockKafkaSupplier.GetProducer(consumeConfig.ToProducerConfig());
            var consumer = mockKafkaSupplier.GetConsumer(consumeConfig.ToConsumerConfig("test-consum"), null);

            consumer.Subscribe("topic2");

            thread1.Start(token1.Token);
            thread2.Start(token2.Token);

            producer.Produce("topic", new Confluent.Kafka.Message <byte[], byte[]>
            {
                Key   = serdes.Serialize("key1", new SerializationContext()),
                Value = serdes.Serialize("coucou", new SerializationContext())
            });
            //WAIT STREAMTHREAD PROCESS MESSAGE

            AssertExtensions.WaitUntil(() => thread1.ActiveTasks.Count() == 2, TimeSpan.FromSeconds(5),
                                       TimeSpan.FromMilliseconds(10));
            AssertExtensions.WaitUntil(() => thread2.ActiveTasks.Count() == 2, TimeSpan.FromSeconds(5),
                                       TimeSpan.FromMilliseconds(10));

            // 2 CONSUMER FOR THE SAME GROUP ID => TOPIC WITH 4 PARTITIONS
            Assert.AreEqual(2, thread1.ActiveTasks.Count());
            Assert.AreEqual(2, thread2.ActiveTasks.Count());

            var message = consumer.Consume(100);

            Assert.AreEqual("key1", serdes.Deserialize(message.Message.Key, new SerializationContext()));
            Assert.AreEqual("coucou", serdes.Deserialize(message.Message.Value, new SerializationContext()));
            // TODO : Finish test with a real cluster Assert.AreEqual(expectedStates, allStates);
        }
        public void ThreadMetricsTest()
        {
            var serdes      = new StringSerDes();
            var cloneConfig = config.Clone();

            cloneConfig.ApplicationId = "consume-test";
            var producer = mockKafkaSupplier.GetProducer(cloneConfig.ToProducerConfig());
            var consumer = mockKafkaSupplier.GetConsumer(cloneConfig.ToConsumerConfig("test-consum"), null);

            consumer.Subscribe("topic2");

            thread.Start(token.Token);

            AssertExtensions.WaitUntil(() => thread.ActiveTasks.Count() == numberPartitions,
                                       TimeSpan.FromSeconds(5),
                                       TimeSpan.FromMilliseconds(100));

            int nbMessage = 1000;

            // produce ${nbMessage} messages to input topic
            for (int i = 0; i < nbMessage; ++i)
            {
                producer.Produce("topic", new Confluent.Kafka.Message <byte[], byte[]>
                {
                    Key   = serdes.Serialize("key" + i, new SerializationContext()),
                    Value = serdes.Serialize("Hi" + i, new SerializationContext())
                });
            }

            var messagesSink = new List <ConsumeResult <byte[], byte[]> >();

            AssertExtensions.WaitUntil(() =>
            {
                messagesSink.AddRange(consumer.ConsumeRecords(TimeSpan.FromSeconds(1)));
                return(messagesSink.Count == nbMessage);
            }, TimeSpan.FromSeconds(5),
                                       TimeSpan.FromMilliseconds(10));

            // waiting end of processing
            Thread.Sleep(1000);

            long now     = DateTime.Now.GetMilliseconds();
            var  sensors = streamMetricsRegistry.GetThreadScopeSensor(threadId);

            var createTaskSensor = sensors.FirstOrDefault(s => s.Name.Equals(GetSensorName(ThreadMetrics.CREATE_TASK)));

            Assert.AreEqual(2, createTaskSensor.Metrics.Count());
            Assert.AreEqual(numberPartitions,
                            createTaskSensor.Metrics[MetricName.NameAndGroup(
                                                         ThreadMetrics.CREATE_TASK + StreamMetricsRegistry.TOTAL_SUFFIX,
                                                         StreamMetricsRegistry.THREAD_LEVEL_GROUP)].Value);
            Assert.IsTrue(
                (double)(createTaskSensor.Metrics[MetricName.NameAndGroup(
                                                      ThreadMetrics.CREATE_TASK + StreamMetricsRegistry.RATE_SUFFIX,
                                                      StreamMetricsRegistry.THREAD_LEVEL_GROUP)].Value) > 0d);

            var closeTaskSensor = sensors.FirstOrDefault(s => s.Name.Equals(GetSensorName(ThreadMetrics.CLOSE_TASK)));

            Assert.AreEqual(2, closeTaskSensor.Metrics.Count());
            Assert.AreEqual(0d,
                            closeTaskSensor.Metrics[MetricName.NameAndGroup(
                                                        ThreadMetrics.CLOSE_TASK + StreamMetricsRegistry.TOTAL_SUFFIX,
                                                        StreamMetricsRegistry.THREAD_LEVEL_GROUP)].Value);
            Assert.AreEqual(0d,
                            closeTaskSensor.Metrics[MetricName.NameAndGroup(
                                                        ThreadMetrics.CLOSE_TASK + StreamMetricsRegistry.RATE_SUFFIX,
                                                        StreamMetricsRegistry.THREAD_LEVEL_GROUP)].Value);

            var commitSensor = sensors.FirstOrDefault(s => s.Name.Equals(GetSensorName(ThreadMetrics.COMMIT)));

            Assert.AreEqual(4, commitSensor.Metrics.Count());
            Assert.IsTrue(
                (double)commitSensor.Metrics[MetricName.NameAndGroup(
                                                 ThreadMetrics.COMMIT + StreamMetricsRegistry.TOTAL_SUFFIX,
                                                 StreamMetricsRegistry.THREAD_LEVEL_GROUP)].Value > 0d);
            Assert.IsTrue(
                (double)commitSensor.Metrics[MetricName.NameAndGroup(
                                                 ThreadMetrics.COMMIT + StreamMetricsRegistry.RATE_SUFFIX,
                                                 StreamMetricsRegistry.THREAD_LEVEL_GROUP)].Value > 0d);
            Assert.IsTrue(
                (double)commitSensor.Metrics[MetricName.NameAndGroup(
                                                 ThreadMetrics.COMMIT + StreamMetricsRegistry.LATENCY_SUFFIX + StreamMetricsRegistry.AVG_SUFFIX,
                                                 StreamMetricsRegistry.THREAD_LEVEL_GROUP)].Value > 0d);
            Assert.IsTrue(
                (double)commitSensor.Metrics[MetricName.NameAndGroup(
                                                 ThreadMetrics.COMMIT + StreamMetricsRegistry.LATENCY_SUFFIX + StreamMetricsRegistry.MAX_SUFFIX,
                                                 StreamMetricsRegistry.THREAD_LEVEL_GROUP)].Value > 0d);

            var pollSensor = sensors.FirstOrDefault(s => s.Name.Equals(GetSensorName(ThreadMetrics.POLL)));

            Assert.AreEqual(4, pollSensor.Metrics.Count());
            Assert.IsTrue(
                (double)pollSensor.Metrics[MetricName.NameAndGroup(
                                               ThreadMetrics.POLL + StreamMetricsRegistry.TOTAL_SUFFIX,
                                               StreamMetricsRegistry.THREAD_LEVEL_GROUP)].Value > 0d);
            Assert.IsTrue(
                (double)pollSensor.Metrics[MetricName.NameAndGroup(
                                               ThreadMetrics.POLL + StreamMetricsRegistry.RATE_SUFFIX,
                                               StreamMetricsRegistry.THREAD_LEVEL_GROUP)].Value > 0d);
            Assert.IsTrue(
                (double)pollSensor.Metrics[MetricName.NameAndGroup(
                                               ThreadMetrics.POLL + StreamMetricsRegistry.LATENCY_SUFFIX + StreamMetricsRegistry.AVG_SUFFIX,
                                               StreamMetricsRegistry.THREAD_LEVEL_GROUP)].Value > 0d);
            Assert.IsTrue(
                (double)pollSensor.Metrics[MetricName.NameAndGroup(
                                               ThreadMetrics.POLL + StreamMetricsRegistry.LATENCY_SUFFIX + StreamMetricsRegistry.MAX_SUFFIX,
                                               StreamMetricsRegistry.THREAD_LEVEL_GROUP)].Value > 0d);

            var pollRecordsSensor = sensors.FirstOrDefault(s => s.Name.Equals(GetSensorName(ThreadMetrics.POLL + StreamMetricsRegistry.RECORDS_SUFFIX)));

            Assert.AreEqual(2, pollRecordsSensor.Metrics.Count());
            Assert.IsTrue(
                (double)pollRecordsSensor.Metrics[MetricName.NameAndGroup(
                                                      ThreadMetrics.POLL + StreamMetricsRegistry.RECORDS_SUFFIX + StreamMetricsRegistry.AVG_SUFFIX,
                                                      StreamMetricsRegistry.THREAD_LEVEL_GROUP)].Value > 0d);
            Assert.IsTrue(
                (double)pollRecordsSensor.Metrics[MetricName.NameAndGroup(
                                                      ThreadMetrics.POLL + StreamMetricsRegistry.RECORDS_SUFFIX + StreamMetricsRegistry.MAX_SUFFIX,
                                                      StreamMetricsRegistry.THREAD_LEVEL_GROUP)].Value > 0d);

            var processRecordsSensor = sensors.FirstOrDefault(s => s.Name.Equals(GetSensorName(ThreadMetrics.PROCESS + StreamMetricsRegistry.RECORDS_SUFFIX)));

            Assert.AreEqual(2, processRecordsSensor.Metrics.Count());
            Assert.IsTrue(
                (double)processRecordsSensor.Metrics[MetricName.NameAndGroup(
                                                         ThreadMetrics.PROCESS + StreamMetricsRegistry.RECORDS_SUFFIX + StreamMetricsRegistry.AVG_SUFFIX,
                                                         StreamMetricsRegistry.THREAD_LEVEL_GROUP)].Value > 0d);
            Assert.IsTrue(
                (double)processRecordsSensor.Metrics[MetricName.NameAndGroup(
                                                         ThreadMetrics.PROCESS + StreamMetricsRegistry.RECORDS_SUFFIX + StreamMetricsRegistry.MAX_SUFFIX,
                                                         StreamMetricsRegistry.THREAD_LEVEL_GROUP)].Value > 0d);

            var processRateSensor = sensors.FirstOrDefault(s => s.Name.Equals(GetSensorName(ThreadMetrics.PROCESS + StreamMetricsRegistry.RATE_SUFFIX)));

            Assert.AreEqual(2, processRateSensor.Metrics.Count());
            Assert.IsTrue(
                (double)processRateSensor.Metrics[MetricName.NameAndGroup(
                                                      ThreadMetrics.PROCESS + StreamMetricsRegistry.RATE_SUFFIX + StreamMetricsRegistry.RATE_SUFFIX,
                                                      StreamMetricsRegistry.THREAD_LEVEL_GROUP)].Value > 0d);
            Assert.IsTrue(
                (double)processRateSensor.Metrics[MetricName.NameAndGroup(
                                                      ThreadMetrics.PROCESS + StreamMetricsRegistry.RATE_SUFFIX + StreamMetricsRegistry.TOTAL_SUFFIX,
                                                      StreamMetricsRegistry.THREAD_LEVEL_GROUP)].Value > 0d);

            var processLatencySensor = sensors.FirstOrDefault(s => s.Name.Equals(GetSensorName(ThreadMetrics.PROCESS + StreamMetricsRegistry.LATENCY_SUFFIX)));

            Assert.AreEqual(2, processLatencySensor.Metrics.Count());
            Assert.IsTrue(
                (double)processLatencySensor.Metrics[MetricName.NameAndGroup(
                                                         ThreadMetrics.PROCESS + StreamMetricsRegistry.LATENCY_SUFFIX + StreamMetricsRegistry.AVG_SUFFIX,
                                                         StreamMetricsRegistry.THREAD_LEVEL_GROUP)].Value > 0d);
            Assert.IsTrue(
                (double)processLatencySensor.Metrics[MetricName.NameAndGroup(
                                                         ThreadMetrics.PROCESS + StreamMetricsRegistry.LATENCY_SUFFIX + StreamMetricsRegistry.MAX_SUFFIX,
                                                         StreamMetricsRegistry.THREAD_LEVEL_GROUP)].Value > 0d);

            // ratio sensors
            var processRatioSensor = sensors.FirstOrDefault(s => s.Name.Equals(GetSensorName(ThreadMetrics.PROCESS + StreamMetricsRegistry.RATIO_SUFFIX)));
            var pollRatioSensor    = sensors.FirstOrDefault(s => s.Name.Equals(GetSensorName(ThreadMetrics.POLL + StreamMetricsRegistry.RATIO_SUFFIX)));
            var commitRatioSensor  = sensors.FirstOrDefault(s => s.Name.Equals(GetSensorName(ThreadMetrics.COMMIT + StreamMetricsRegistry.RATIO_SUFFIX)));

            Assert.AreEqual(1, processRatioSensor.Metrics.Count());
            Assert.AreEqual(1, pollRatioSensor.Metrics.Count());
            Assert.AreEqual(1, commitRatioSensor.Metrics.Count());

            var processRatioValue = (double)processRatioSensor.Metrics[MetricName.NameAndGroup(
                                                                           ThreadMetrics.PROCESS + StreamMetricsRegistry.RATIO_SUFFIX,
                                                                           StreamMetricsRegistry.THREAD_LEVEL_GROUP)].Value;

            var pollRatioValue = (double)pollRatioSensor.Metrics[MetricName.NameAndGroup(
                                                                     ThreadMetrics.POLL + StreamMetricsRegistry.RATIO_SUFFIX,
                                                                     StreamMetricsRegistry.THREAD_LEVEL_GROUP)].Value;

            var commitRatioValue = (double)commitRatioSensor.Metrics[MetricName.NameAndGroup(
                                                                         ThreadMetrics.COMMIT + StreamMetricsRegistry.RATIO_SUFFIX,
                                                                         StreamMetricsRegistry.THREAD_LEVEL_GROUP)].Value;

            double total = Math.Round(processRatioValue + pollRatioValue + commitRatioValue, 2);

            // we accept 10% of lost
            Assert.IsTrue(total >= 0.90d);
        }
        public void StreamThreadNormalWorkflowWithRebalancing()
        {
            List <ThreadState> allStates = new List <ThreadState>();
            var expectedStates           = new List <ThreadState>
            {
                ThreadState.CREATED,
                ThreadState.STARTING,
                ThreadState.PARTITIONS_ASSIGNED,
                ThreadState.RUNNING,
                ThreadState.PARTITIONS_REVOKED,
                ThreadState.PARTITIONS_ASSIGNED,
                ThreadState.RUNNING,
                ThreadState.PENDING_SHUTDOWN,
                ThreadState.DEAD
            };

            var source = new System.Threading.CancellationTokenSource();
            var config = new StreamConfig <StringSerDes, StringSerDes>();

            config.ApplicationId = "test";
            config.Guarantee     = ProcessingGuarantee.AT_LEAST_ONCE;
            config.PollMs        = 1;

            var consumeConfig = config.Clone();

            consumeConfig.ApplicationId = "consume-test";

            var serdes  = new StringSerDes();
            var builder = new StreamBuilder();

            builder.Stream <string, string>("topic").To("topic2");

            var topo = builder.Build();

            var supplier = new MockKafkaSupplier(4);
            var producer = supplier.GetProducer(consumeConfig.ToProducerConfig());
            var consumer = supplier.GetConsumer(consumeConfig.ToConsumerConfig("test-consum"), null);

            consumer.Subscribe("topic2");

            var thread = StreamThread.Create(
                "thread-0", "c0",
                topo.Builder, config,
                supplier, supplier.GetAdmin(config.ToAdminConfig("admin")),
                0) as StreamThread;

            allStates.Add(thread.State);
            thread.StateChanged += (t, o, n) =>
            {
                Assert.IsInstanceOf <ThreadState>(n);
                allStates.Add(n as ThreadState);
            };

            thread.Start(source.Token);
            // WAIT PARTITONS ASSIGNED
            System.Threading.Thread.Sleep(50);

            var thread2 = StreamThread.Create(
                "thread-1", "c1",
                topo.Builder, config,
                supplier, supplier.GetAdmin(config.ToAdminConfig("admin")),
                1) as StreamThread;

            thread2.Start(source.Token);
            // WAIT PARTITONS REBALANCING
            System.Threading.Thread.Sleep(50);

            producer.Produce("topic", new Message <byte[], byte[]>
            {
                Key   = serdes.Serialize("key1", new SerializationContext()),
                Value = serdes.Serialize("coucou", new SerializationContext())
            });
            //WAIT STREAMTHREAD PROCESS MESSAGE
            System.Threading.Thread.Sleep(100);
            var message = consumer.Consume(100);

            // 2 CONSUMER FOR THE SAME GROUP ID => TOPIC WITH 4 PARTITIONS
            Assert.AreEqual(2, thread.ActiveTasks.Count());
            Assert.AreEqual(2, thread2.ActiveTasks.Count());

            source.Cancel();
            thread.Dispose();
            thread2.Dispose();

            Assert.AreEqual("key1", serdes.Deserialize(message.Message.Key, new SerializationContext()));
            Assert.AreEqual("coucou", serdes.Deserialize(message.Message.Value, new SerializationContext()));
            Assert.AreEqual(expectedStates, allStates);
            // Destroy in memory cluster
            supplier.Destroy();
        }
Beispiel #7
0
        public void StreamThreadRestorationPhaseStartDifferent()
        {
            var producerConfig = config.Clone();

            producerConfig.ApplicationId = "produce-test";

            var serdes   = new StringSerDes();
            var producer = mockKafkaSupplier.GetProducer(producerConfig.ToProducerConfig());

            thread1.Start(token1.Token);
            Thread.Sleep(1500);
            thread2.Start(token2.Token);

            producer.Produce(new TopicPartition("topic", 0), new Confluent.Kafka.Message <byte[], byte[]>
            {
                Key   = serdes.Serialize("key1", new SerializationContext()),
                Value = serdes.Serialize("coucou", new SerializationContext())
            });
            producer.Produce(new TopicPartition("topic", 1), new Confluent.Kafka.Message <byte[], byte[]>
            {
                Key   = serdes.Serialize("key2", new SerializationContext()),
                Value = serdes.Serialize("coucou", new SerializationContext())
            });

            AssertExtensions.WaitUntil(
                () => thread1.State == ThreadState.RUNNING &&
                thread2.State == ThreadState.RUNNING,
                TimeSpan.FromSeconds(5),
                TimeSpan.FromMilliseconds(20));

            // 2 CONSUMER FOR THE SAME GROUP ID => TOPIC WITH 2 PARTITIONS
            Assert.AreEqual(1, thread1.ActiveTasks.Count());
            Assert.AreEqual(1, thread2.ActiveTasks.Count());

            AssertExtensions.WaitUntil(
                () => thread1.ActiveTasks.ToList()[0].State == TaskState.RUNNING &&
                thread2.ActiveTasks.ToList()[0].State == TaskState.RUNNING,
                TimeSpan.FromSeconds(5),
                TimeSpan.FromMilliseconds(20));

            var storeThread1 =
                thread1.ActiveTasks.ToList()[0].GetStore("store") as ITimestampedKeyValueStore <string, string>;
            var storeThread2 =
                thread2.ActiveTasks.ToList()[0].GetStore("store") as ITimestampedKeyValueStore <string, string>;

            Assert.IsNotNull(storeThread1);
            Assert.IsNotNull(storeThread2);

            AssertExtensions.WaitUntil(
                () => storeThread1.All().ToList().Count == 1,
                TimeSpan.FromSeconds(5),
                TimeSpan.FromMilliseconds(20));

            AssertExtensions.WaitUntil(
                () => storeThread2.All().ToList().Count == 1,
                TimeSpan.FromSeconds(5),
                TimeSpan.FromMilliseconds(20));

            var totalItemsSt1 = storeThread1.All().ToList();
            var totalItemsSt2 = storeThread2.All().ToList();

            Assert.AreEqual(1, totalItemsSt1.Count);
            Assert.AreEqual(1, totalItemsSt2.Count);

            // Thread2 closed, partitions assigned from thread2 rebalance to thread1
            // Thread1 need to restore state store
            token2.Cancel();
            thread2.Dispose();

            thread2Disposed = true;

            producer.Produce(new TopicPartition("topic", 1), new Confluent.Kafka.Message <byte[], byte[]>
            {
                Key   = serdes.Serialize("key3", new SerializationContext()),
                Value = serdes.Serialize("coucou", new SerializationContext())
            });

            AssertExtensions.WaitUntil(
                () => thread1.State == ThreadState.RUNNING && thread1.ActiveTasks.Count() == 2,
                TimeSpan.FromSeconds(5),
                TimeSpan.FromMilliseconds(20));

            Assert.AreEqual(2, thread1.ActiveTasks.Count());

            AssertExtensions.WaitUntil(
                () => thread1.ActiveTasks.ToList()[0].State == TaskState.RUNNING &&
                thread1.ActiveTasks.ToList()[1].State == TaskState.RUNNING,
                TimeSpan.FromSeconds(5),
                TimeSpan.FromMilliseconds(20));

            var storeThreadTask1 =
                thread1.ActiveTasks.ToList()[0].GetStore("store") as ITimestampedKeyValueStore <string, string>;
            var storeThreadTask2 =
                thread1.ActiveTasks.ToList()[1].GetStore("store") as ITimestampedKeyValueStore <string, string>;

            Assert.IsNotNull(storeThreadTask1);
            Assert.IsNotNull(storeThreadTask2);

            bool task0Part0 = thread1.ActiveTasks.ToList()[0].Id.Partition == 0;

            AssertExtensions.WaitUntil(
                () => storeThreadTask1.All().ToList().Count == (task0Part0 ? 1 : 2),
                TimeSpan.FromSeconds(1),
                TimeSpan.FromMilliseconds(20));

            AssertExtensions.WaitUntil(
                () => storeThreadTask2.All().ToList().Count == (task0Part0 ? 2 : 1),
                TimeSpan.FromSeconds(1),
                TimeSpan.FromMilliseconds(20));

            var totalItemsSt10 = storeThreadTask1.All().ToList();
            var totalItemsSt11 = storeThreadTask2.All().ToList();

            Assert.AreEqual((task0Part0 ? 1 : 2), totalItemsSt10.Count);
            Assert.AreEqual((task0Part0 ? 2 : 1), totalItemsSt11.Count);
        }
Beispiel #8
0
        public void TestConsumeSaveOffset()
        {
            var consumerConfig = new ConsumerConfig();

            consumerConfig.GroupId  = "cg";
            consumerConfig.ClientId = "cg-0";

            var consumerConfig1 = new ConsumerConfig();

            consumerConfig1.GroupId  = "cg";
            consumerConfig1.ClientId = "cg-1";

            var supplier = new MockKafkaSupplier(2);
            var producer = supplier.GetProducer(new ProducerConfig());

            producer.Produce(new TopicPartition("topic", 0), new Message <byte[], byte[]> {
                Key = new byte[1] {
                    42
                }, Value = new byte[1] {
                    12
                }
            });
            producer.Produce(new TopicPartition("topic", 1), new Message <byte[], byte[]> {
                Key = new byte[1] {
                    43
                }, Value = new byte[1] {
                    13
                }
            });

            var c1 = supplier.GetConsumer(consumerConfig, null);
            var c2 = supplier.GetConsumer(consumerConfig1, null);

            c1.Subscribe(new List <string> {
                "topic"
            });
            c1.Consume();
            c2.Subscribe(new List <string> {
                "topic"
            });
            c2.Consume();

            c1.Consume();
            c2.Consume();

            Assert.AreEqual(1, c1.Assignment.Count);
            Assert.AreEqual(1, c2.Assignment.Count);

            c1.Unsubscribe();
            c2.Unsubscribe();

            c1.Subscribe(new List <string> {
                "topic"
            });
            c1.Consume();
            c2.Subscribe(new List <string> {
                "topic"
            });
            c2.Consume();

            c2.Consume();
            c1.Consume();
            c2.Consume();

            Assert.AreEqual(1, c1.Assignment.Count);
            Assert.AreEqual(1, c2.Assignment.Count);
        }