MultiFetch() public method

Combine multiple fetch requests in one call.
Offset is passed in on every request, allowing the user to maintain this metadata however they choose.
public MultiFetch ( MultiFetchRequest request ) : IList
request Kafka.Client.Requests.MultiFetchRequest /// The list of fetch requests. ///
return IList
Consumer Class Documentation
Example #1
0
        public void ProducerSendsAndConsumerReceivesMultiRequest()
        {
            var prodConfig = this.SyncProducerConfig1;
            var consumerConfig = this.ConsumerConfig1;

            string testTopic1 = CurrentTestTopic + "1";
            string testTopic2 = CurrentTestTopic + "2";
            string testTopic3 = CurrentTestTopic + "3";

            var sourceMessage1 = new Message(Encoding.UTF8.GetBytes("1: TestMessage"));
            var sourceMessage2 = new Message(Encoding.UTF8.GetBytes("2: TestMessage"));
            var sourceMessage3 = new Message(Encoding.UTF8.GetBytes("3: TestMessage"));
            var sourceMessage4 = new Message(Encoding.UTF8.GetBytes("4: TestMessage"));

            var requests = new List<ProducerRequest>
            {
                new ProducerRequest(testTopic1, 0, new List<Message> { sourceMessage1 }),
                new ProducerRequest(testTopic1, 0, new List<Message> { sourceMessage2 }),
                new ProducerRequest(testTopic2, 0, new List<Message> { sourceMessage3 }),
                new ProducerRequest(testTopic3, 0, new List<Message> { sourceMessage4 })
            };

            long currentOffset1 = TestHelper.GetCurrentKafkaOffset(testTopic1, consumerConfig);
            long currentOffset2 = TestHelper.GetCurrentKafkaOffset(testTopic2, consumerConfig);
            long currentOffset3 = TestHelper.GetCurrentKafkaOffset(testTopic3, consumerConfig);

            using (var producer = new SyncProducer(prodConfig))
            {
                producer.MultiSend(requests);
            }

            IConsumer consumer = new Consumer(consumerConfig);
            var request = new MultiFetchRequest(new List<FetchRequest>
            {
                new FetchRequest(testTopic1, 0, currentOffset1),
                new FetchRequest(testTopic2, 0, currentOffset2),
                new FetchRequest(testTopic3, 0, currentOffset3)
            });
            IList<BufferedMessageSet> messageSets;
            int totalWaitTimeInMiliseconds = 0;
            int waitSingle = 100;
            while (true)
            {
                Thread.Sleep(waitSingle);
                messageSets = consumer.MultiFetch(request);
                if (messageSets.Count > 2 && messageSets[0].Messages.Count() > 0 && messageSets[1].Messages.Count() > 0 && messageSets[2].Messages.Count() > 0)
                {
                    break;
                }

                totalWaitTimeInMiliseconds += waitSingle;
                if (totalWaitTimeInMiliseconds >= MaxTestWaitTimeInMiliseconds)
                {
                    break;
                }
            }

            Assert.AreEqual(3, messageSets.Count);
            Assert.AreEqual(2, messageSets[0].Messages.Count());
            Assert.AreEqual(1, messageSets[1].Messages.Count());
            Assert.AreEqual(1, messageSets[2].Messages.Count());
            Assert.AreEqual(sourceMessage1.ToString(), messageSets[0].Messages.First().ToString());
            Assert.AreEqual(sourceMessage2.ToString(), messageSets[0].Messages.Skip(1).First().ToString());
            Assert.AreEqual(sourceMessage3.ToString(), messageSets[1].Messages.First().ToString());
            Assert.AreEqual(sourceMessage4.ToString(), messageSets[2].Messages.First().ToString());
        }
Example #2
0
        public void ConsumerMultiFetchGetsMessage()
        {
            var config = this.ConsumerConfig1;

            ProducerSendMultiRequest();
            Thread.Sleep(2000);
            IConsumer cons = new Consumer(config);
            var request = new MultiFetchRequest(new List<FetchRequest>
            {
                new FetchRequest(CurrentTestTopic, 0, 0),
                new FetchRequest(CurrentTestTopic, 0, 0),
                new FetchRequest(CurrentTestTopic, 0, 0)
            });

            IList<BufferedMessageSet> response = cons.MultiFetch(request);
            Assert.AreEqual(3, response.Count);
            for (int ix = 0; ix < response.Count; ix++)
            {
                IEnumerable<Message> messageSet = response[ix].Messages;
                Assert.AreEqual(4, messageSet.Count());
                Console.WriteLine(string.Format("Request #{0}-->", ix));
                foreach (Message msg in messageSet)
                {
                    Console.WriteLine(msg.ToString());
                }
            }
        }