The low-level API of consumer of Kafka messages
Maintains a connection to a single broker and has a close correspondence to the network requests sent to the server. Also, is completely stateless.
Наследование: IConsumer
        public void Start()
        {
            while (!_stop)
            {
                try
                {
                    if (_partitionsNeedingLeader.IsEmpty)
                    {
                        Thread.Sleep(_config.ConsumeGroupFindNewLeaderSleepIntervalMs);
                        continue;
                    }

                    PartitionTopicInfo partition;
                    if (_partitionsNeedingLeader.TryDequeue(out partition))
                    {
                        Logger.DebugFormat("Finding new leader for topic {0}, partition {1}", partition.Topic, partition.PartitionId);
                        Broker newLeader = null;
                        foreach (KeyValuePair<int, Broker> broker in _brokers.Brokers)
                        {
                            var consumer = new Consumer(_config, broker.Value.Host, broker.Value.Port);
                            try
                            {
                                IEnumerable<TopicMetadata> metaData = consumer.GetMetaData(TopicMetadataRequest.Create(new[] { partition.Topic }, 1, 0, clientId));
                                if (metaData != null && metaData.Any())
                                {
                                    PartitionMetadata newPartitionData = metaData.First().PartitionsMetadata.FirstOrDefault(p => p.PartitionId == partition.PartitionId);
                                    if (newPartitionData != null)
                                    {
                                        Logger.DebugFormat("New leader for {0} ({1}) is broker {2}", partition.Topic, partition.PartitionId, newPartitionData.Leader.Id);
                                        newLeader = newPartitionData.Leader;
                                        break;
                                    }
                                }
                            }
                            catch (Exception ex)
                            {
                                Logger.WarnFormat("Error retrieving meta data from broker {0}: {1}", broker.Value.Id, ex.FormatException());
                            }
                        }

                        if (newLeader == null)
                        {
                            Logger.ErrorFormat("New leader information could not be retrieved for {0} ({1})", partition.Topic, partition.PartitionId);
                        }
                        else
                        {
                            _createNewFetcher(partition, newLeader);
                        }
                    }
                }
                catch (Exception ex)
                {
                    Logger.ErrorFormat("PartitionLeaderFinder encountered an error: {0}", ex.FormatException());
                    Thread.Sleep(FailureRetryDelayMs);
                }
            }

            Logger.Info("Partition leader finder thread shutting down.");
        }
 /// <summary>
 /// Retrive first or last offset for a given partition.
 /// </summary>
 /// <remarks>
 /// If <see cref="offsetRequestConstant"/> not equal to <see cref="OffsetRequest.LatestTime"/> or <see cref="OffsetRequest.EarliestTime"/> method returns offset before specified number.
 /// </remarks>
 /// <param name="consumer">Consumer instance</param>
 /// <param name="topic">The topic</param>
 /// <param name="partitionId">Partition Id</param>
 /// <param name="offsetRequestConstant">Offset that indicates what offset need to return.</param>
 /// <returns>
 /// Retrive first or last offset for a given partition based on <see cref="offsetRequestConstant"/> parameter.
 /// If offset couldn't be retrieved returns null.
 /// </returns>
 public static long? EarliestOrLatestOffset(Consumer consumer, string topic, int partitionId, long offsetRequestConstant)
 {
     var requestInfos = new Dictionary<string, List<PartitionOffsetRequestInfo>>();
     requestInfos[topic] = new List<PartitionOffsetRequestInfo>() { new PartitionOffsetRequestInfo(partitionId, offsetRequestConstant, 1) };
     var offsets = consumer.GetOffsetsBefore(new OffsetRequest(requestInfos));
     var topicResult = offsets.ResponseMap[topic];
     var responseRow = topicResult != null ? topicResult.FirstOrDefault() : null;
     return responseRow != null && responseRow.Offsets != null && responseRow.Offsets.Count > 0 ? responseRow.Offsets[0] : (long?)null;
 }
Пример #3
0
        public void AsyncProducerSendsAndConsumerReceivesSingleSimpleMessage()
        {
            var prodConfig = this.AsyncProducerConfig1;
            var consumerConfig = this.ConsumerConfig1;

            var sourceMessage = new Message(Encoding.UTF8.GetBytes("test message"));
            using (var producer = new AsyncProducer(prodConfig))
            {
                var producerRequest = new ProducerRequest(CurrentTestTopic, 0, new List<Message> { sourceMessage });
                producer.Send(producerRequest);
            }

            long currentOffset = TestHelper.GetCurrentKafkaOffset(CurrentTestTopic, consumerConfig);
            IConsumer consumer = new Consumer(consumerConfig);
            var request = new FetchRequest(CurrentTestTopic, 0, currentOffset);

            BufferedMessageSet response;
            int totalWaitTimeInMiliseconds = 0;
            int waitSingle = 100;
            while (true)
            {
                Thread.Sleep(waitSingle);
                response = consumer.Fetch(request);
                if (response != null && response.Messages.Count() > 0)
                {
                    break;
                }

                totalWaitTimeInMiliseconds += waitSingle;
                if (totalWaitTimeInMiliseconds >= MaxTestWaitTimeInMiliseconds)
                {
                    break;
                }
            }

            Assert.NotNull(response);
            Assert.AreEqual(1, response.Messages.Count());
            Message resultMessage = response.Messages.First();
            Assert.AreEqual(sourceMessage.ToString(), resultMessage.ToString());
        }
Пример #4
0
        public void ConsumerFetchMessage()
        {
            ProducerSendsMessage();

            ConsumerConfig config = new ConsumerConfig(clientConfig);
            IConsumer consumer = new Kafka.Client.Consumers.Consumer(config);
            FetchRequest request = new FetchRequest(CurrentTestTopic, 0, 0);
            BufferedMessageSet response = consumer.Fetch(request);
            Assert.NotNull(response);
            foreach (var message in response.Messages)
            {
                Console.WriteLine(message);
            }
        }
Пример #5
0
        public void ProducerSendsAndConsumerReceivesMultiRequest()
        {
            var prodConfig = this.SyncProducerConfig1;
            var consumerConfig = this.ConsumerConfig1;

            string testTopic1 = CurrentTestTopic + "1";
            string testTopic2 = CurrentTestTopic + "2";
            string testTopic3 = CurrentTestTopic + "3";

            var sourceMessage1 = new Message(Encoding.UTF8.GetBytes("1: TestMessage"));
            var sourceMessage2 = new Message(Encoding.UTF8.GetBytes("2: TestMessage"));
            var sourceMessage3 = new Message(Encoding.UTF8.GetBytes("3: TestMessage"));
            var sourceMessage4 = new Message(Encoding.UTF8.GetBytes("4: TestMessage"));

            var requests = new List<ProducerRequest>
            {
                new ProducerRequest(testTopic1, 0, new List<Message> { sourceMessage1 }),
                new ProducerRequest(testTopic1, 0, new List<Message> { sourceMessage2 }),
                new ProducerRequest(testTopic2, 0, new List<Message> { sourceMessage3 }),
                new ProducerRequest(testTopic3, 0, new List<Message> { sourceMessage4 })
            };

            long currentOffset1 = TestHelper.GetCurrentKafkaOffset(testTopic1, consumerConfig);
            long currentOffset2 = TestHelper.GetCurrentKafkaOffset(testTopic2, consumerConfig);
            long currentOffset3 = TestHelper.GetCurrentKafkaOffset(testTopic3, consumerConfig);

            using (var producer = new SyncProducer(prodConfig))
            {
                producer.MultiSend(requests);
            }

            IConsumer consumer = new Consumer(consumerConfig);
            var request = new MultiFetchRequest(new List<FetchRequest>
            {
                new FetchRequest(testTopic1, 0, currentOffset1),
                new FetchRequest(testTopic2, 0, currentOffset2),
                new FetchRequest(testTopic3, 0, currentOffset3)
            });
            IList<BufferedMessageSet> messageSets;
            int totalWaitTimeInMiliseconds = 0;
            int waitSingle = 100;
            while (true)
            {
                Thread.Sleep(waitSingle);
                messageSets = consumer.MultiFetch(request);
                if (messageSets.Count > 2 && messageSets[0].Messages.Count() > 0 && messageSets[1].Messages.Count() > 0 && messageSets[2].Messages.Count() > 0)
                {
                    break;
                }

                totalWaitTimeInMiliseconds += waitSingle;
                if (totalWaitTimeInMiliseconds >= MaxTestWaitTimeInMiliseconds)
                {
                    break;
                }
            }

            Assert.AreEqual(3, messageSets.Count);
            Assert.AreEqual(2, messageSets[0].Messages.Count());
            Assert.AreEqual(1, messageSets[1].Messages.Count());
            Assert.AreEqual(1, messageSets[2].Messages.Count());
            Assert.AreEqual(sourceMessage1.ToString(), messageSets[0].Messages.First().ToString());
            Assert.AreEqual(sourceMessage2.ToString(), messageSets[0].Messages.Skip(1).First().ToString());
            Assert.AreEqual(sourceMessage3.ToString(), messageSets[1].Messages.First().ToString());
            Assert.AreEqual(sourceMessage4.ToString(), messageSets[2].Messages.First().ToString());
        }
Пример #6
0
        public void ConsumerMultiFetchGetsMessage()
        {
            var config = this.ConsumerConfig1;

            ProducerSendMultiRequest();
            Thread.Sleep(2000);
            IConsumer cons = new Consumer(config);
            var request = new MultiFetchRequest(new List<FetchRequest>
            {
                new FetchRequest(CurrentTestTopic, 0, 0),
                new FetchRequest(CurrentTestTopic, 0, 0),
                new FetchRequest(CurrentTestTopic, 0, 0)
            });

            IList<BufferedMessageSet> response = cons.MultiFetch(request);
            Assert.AreEqual(3, response.Count);
            for (int ix = 0; ix < response.Count; ix++)
            {
                IEnumerable<Message> messageSet = response[ix].Messages;
                Assert.AreEqual(4, messageSet.Count());
                Console.WriteLine(string.Format("Request #{0}-->", ix));
                foreach (Message msg in messageSet)
                {
                    Console.WriteLine(msg.ToString());
                }
            }
        }
Пример #7
0
        public void ConsumerGetsOffsets()
        {
            var consumerConfig = this.ConsumerConfig1;

            var request = new OffsetRequest(CurrentTestTopic, 0, DateTime.Now.AddHours(-24).Ticks, 10);
            IConsumer consumer = new Consumer(consumerConfig);
            IList<long> list = consumer.GetOffsetsBefore(request);

            foreach (long l in list)
            {
                Console.Out.WriteLine(l);
            }
        }
Пример #8
0
        public void ConsumerFetchMessage()
        {
            var consumerConfig = this.ConsumerConfig1;
            ProducerSendsMessage();
            Thread.Sleep(1000);
            IConsumer consumer = new Consumer(consumerConfig);
            var request = new FetchRequest(CurrentTestTopic, 0, 0);
            BufferedMessageSet response = consumer.Fetch(request);
            Assert.NotNull(response);
            int count = 0;
            foreach (var message in response)
            {
                count++;
                Console.WriteLine(message.Message);
            }

            Assert.AreEqual(2, count);
        }