public ZKSessionExpireListener(ZookeeperConsumerConnector parent, ZKGroupDirs dirs, string consumerIdString, TopicCount topicCount, IZKRebalancerListener loadbalancerListener) { this.parent = parent; this.Dirs = dirs; this.ConsumerIdString = consumerIdString; this.TopicCount = topicCount; this.LoadbalancerListener = loadbalancerListener; }
public ZKSessionExpireListener(ZKGroupDirs dirs, string consumerIdString, TopicCount topicCount, ZKRebalancerListener loadBalancerListener, ZookeeperConsumerConnector zkConsumerConnector) { this.consumerIdString = consumerIdString; this.loadBalancerListener = loadBalancerListener; this.zkConsumerConnector = zkConsumerConnector; this.dirs = dirs; this.topicCount = topicCount; }
public ZKRebalancerListener( ZookeeperConsumerConnector parent, string group, string consumerIdString, IDictionary <string, IList <KafkaStream <TKey, TValue> > > kafkaMessageAndMetadataStreams) { this.parent = parent; this.group = group; this.consumerIdString = consumerIdString; this.KafkaMessageAndMetadataStreams = kafkaMessageAndMetadataStreams; this.@lock = new ReentrantLock(); this.cond = [email protected](); this.watcherExecutorThread = new Thread(() => { Logger.InfoFormat("starting watcher executor thread for consumer {0}", consumerIdString); bool doRebalance; while (!parent.isShuttingDown.Get()) { try { [email protected](); try { if (!isWatcherTriggered) { cond.Await(TimeSpan.FromMilliseconds(1000)); // wake up periodically so that it can check the shutdown flag } } finally { doRebalance = isWatcherTriggered; isWatcherTriggered = false; @lock.Unlock(); } if (doRebalance) { this.SyncedRebalance(); } } catch (Exception e) { Logger.Error("Error during syncedRebalance", e); } } Logger.InfoFormat("Stoppping watcher executer thread for consumer {0}", consumerIdString); }); this.watcherExecutorThread.Name = consumerIdString + "_watcher_executor"; this.watcherExecutorThread.Start(); }
public void ConsumerPerformsRebalancingWhenConsumerIsRemovedAndTakesItsPartitions() { var config = new ConsumerConfig(clientConfig) { AutoCommit = false, GroupId = "group1", ZkSessionTimeoutMs = 60000, ZkConnectionTimeoutMs = 60000 }; IList<string> ids; IList<string> owners; using (var consumerConnector = new ZookeeperConsumerConnector(config, true)) { var client = ReflectionHelper.GetInstanceField<ZooKeeperClient>("zkClient", consumerConnector); Assert.IsNotNull(client); client.DeleteRecursive("/consumers/group1"); var topicCount = new Dictionary<string, int> { { "test", 1 } }; consumerConnector.CreateMessageStreams(topicCount); WaitUntillIdle(client, 1000); using (var consumerConnector2 = new ZookeeperConsumerConnector(config, true)) { consumerConnector2.CreateMessageStreams(topicCount); WaitUntillIdle(client, 1000); ids = client.GetChildren("/consumers/group1/ids", false).ToList(); owners = client.GetChildren("/consumers/group1/owners/test", false).ToList(); Assert.That(ids, Is.Not.Null.And.Not.Empty); Assert.That(ids.Count, Is.EqualTo(2)); Assert.That(owners, Is.Not.Null.And.Not.Empty); Assert.That(owners.Count, Is.EqualTo(2)); } WaitUntillIdle(client, 1000); ids = client.GetChildren("/consumers/group1/ids", false).ToList(); owners = client.GetChildren("/consumers/group1/owners/test", false).ToList(); Assert.That(ids, Is.Not.Null.And.Not.Empty); Assert.That(ids.Count, Is.EqualTo(1)); Assert.That(owners, Is.Not.Null.And.Not.Empty); Assert.That(owners.Count, Is.EqualTo(2)); var data1 = client.ReadData<string>("/consumers/group1/owners/test/" + owners[0], false); var data2 = client.ReadData<string>("/consumers/group1/owners/test/" + owners[1], false); Assert.That(data1, Is.Not.Null.And.Not.Empty); Assert.That(data2, Is.Not.Null.And.Not.Empty); Assert.That(data1, Is.EqualTo(data2)); Assert.That(data1, Is.StringStarting(ids[0])); } }
internal WildcardStreamsHandler( ZookeeperConsumerConnector parent, TopicFilter topicFilter, int numStreams, IDecoder <TKey> keyDecoder, IDecoder <TValue> valueDecoder) { this.parent = parent; this.topicFilter = topicFilter; this.numStreams = numStreams; this.keyDecoder = keyDecoder; this.valueDecoder = valueDecoder; if (parent.messageStreamCreated.GetAndSet(true)) { throw new Exception("Each consumer connector can create message streams by filter at most once."); } this.wildcardQueuesAndStreams = Enumerable.Range(1, numStreams).Select(e => { var queue = new BlockingCollection <FetchedDataChunk>(this.parent.Config.QueuedMaxMessages); var stream = new KafkaStream <TKey, TValue>( queue, this.parent.Config.ConsumerTimeoutMs, keyDecoder, valueDecoder, this.parent.Config.ClientId); return(Tuple.Create(queue, stream)); }).ToList(); this.wildcardTopics = ZkUtils.GetChildrenParentMayNotExist(this.parent.zkClient, ZkUtils.BrokerTopicsPath) .Where(topicFilter.IsTopicAllowed) .ToList(); this.wildcardTopicCount = TopicCount.ConstructTopicCount( this.parent.consumerIdString, topicFilter, numStreams, this.parent.zkClient); this.dirs = new ZKGroupDirs(this.parent.Config.GroupId); this.parent.RegisterConsumerInZK(dirs, this.parent.consumerIdString, this.wildcardTopicCount); this.parent.ReinitializeConsumer(this.wildcardTopicCount, this.wildcardQueuesAndStreams); // Topic events will trigger subsequent synced rebalances. Logger.InfoFormat("Creating topic event watcher for topics {0}", topicFilter); this.parent.wildcardTopicWatcher = new ZookeeperTopicEventWatcher(this.parent.zkClient, this); }
internal ZKRebalancerListener( ConsumerConfiguration config, string consumerIdString, IDictionary<string, IDictionary<Partition, PartitionTopicInfo>> topicRegistry, IZooKeeperClient zkClient, ZookeeperConsumerConnector zkConsumerConnector, IDictionary<Tuple<string, string>, BlockingCollection<FetchedDataChunk>> queues, Fetcher fetcher, object syncLock) { this.syncLock = syncLock; this.consumerIdString = consumerIdString; this.config = config; this.topicRegistry = topicRegistry; this.zkClient = zkClient; this.dirs = new ZKGroupDirs(config.GroupId); this.zkConsumerConnector = zkConsumerConnector; this.queues = queues; this.fetcher = fetcher; }
public void ConsumerPerformsRebalancingWhenNewConsumerIsAddedAndTheyDividePartitions() { var config = this.ZooKeeperBasedConsumerConfig; IList<string> ids; IList<string> owners; using (var consumerConnector = new ZookeeperConsumerConnector(config, true)) { var client = ReflectionHelper.GetInstanceField<ZooKeeperClient>( "zkClient", consumerConnector); Assert.IsNotNull(client); client.DeleteRecursive("/consumers/group1"); var topicCount = new Dictionary<string, int> { { "test", 1 } }; consumerConnector.CreateMessageStreams(topicCount); WaitUntillIdle(client, 1000); using (var consumerConnector2 = new ZookeeperConsumerConnector(config, true)) { consumerConnector2.CreateMessageStreams(topicCount); WaitUntillIdle(client, 1000); ids = client.GetChildren("/consumers/group1/ids", false).ToList(); owners = client.GetChildren("/consumers/group1/owners/test", false).ToList(); Assert.That(ids, Is.Not.Null.And.Not.Empty); Assert.That(ids.Count, Is.EqualTo(2)); Assert.That(owners, Is.Not.Null.And.Not.Empty); Assert.That(owners.Count, Is.EqualTo(2)); var data1 = client.ReadData<string>("/consumers/group1/owners/test/" + owners[0], false); var data2 = client.ReadData<string>("/consumers/group1/owners/test/" + owners[1], false); Assert.That(data1, Is.Not.Null.And.Not.Empty); Assert.That(data2, Is.Not.Null.And.Not.Empty); Assert.That(data1, Is.Not.EqualTo(data2)); Assert.That(data1, Is.StringStarting(ids[0]).Or.StringStarting(ids[1])); Assert.That(data2, Is.StringStarting(ids[0]).Or.StringStarting(ids[1])); } } }
public void TestBasic() { // test consumer timeout logic var consumerConfig0 = TestUtils.CreateConsumerProperties(ZkConnect, Group, Consumer0, 200); var zkConsumerConnector0 = new ZookeeperConsumerConnector(consumerConfig0); var topicMessageSterams0 = zkConsumerConnector0.CreateMessageStreams( new Dictionary<string, int> { { Topic, 1 } }, new StringDecoder(), new StringDecoder()); // no messages to consume, we should hit timeout; // also the iterator should support re-entrant, so loop it twice for (var i = 0; i < 2; i++) { Assert.Throws<ConsumerTimeoutException>( () => this.GetMessages(nMessages * 2, topicMessageSterams0)); } zkConsumerConnector0.Shutdown(); // send some messages to each broker var sentMessages1 = this.SendMessagesToBrokerPartition(Configs.First(), Topic, 0, nMessages) .Union(this.SendMessagesToBrokerPartition(Configs.Last(), Topic, 0, nMessages)).ToList(); // wait to make sure the topic and partition have a leader for the successful case TestUtils.WaitUntilLeaderIsElectedOrChanged(this.ZkClient, Topic, 0, 500); TestUtils.WaitUntilLeaderIsElectedOrChanged(this.ZkClient, Topic, 1, 500); TestUtils.WaitUntilMetadataIsPropagated(this.Servers, Topic, 0, 1000); TestUtils.WaitUntilMetadataIsPropagated(this.Servers, Topic, 1, 1000); // create a consuemr var consumerConfig1 = TestUtils.CreateConsumerProperties(ZkConnect, Group, Consumer1); var zkConsumerConnector1 = new ZookeeperConsumerConnector(consumerConfig1); var topicMessageStreams1 = zkConsumerConnector1.CreateMessageStreams( new Dictionary<string, int> { { Topic, 1 } }, new StringDecoder(), new StringDecoder()); var receivedMessages1 = this.GetMessages(nMessages * 2, topicMessageStreams1); Assert.Equal(sentMessages1.OrderBy(x => x).ToArray(), receivedMessages1.OrderBy(x => x).ToArray()); // also check partition ownership var actual_1 = this.GetZKChildrenValues(this.dirs.ConsumerOwnerDir); var expected_1 = new List<Tuple<string, string>> { Tuple.Create("0", "group1_consumer1-0"), Tuple.Create("1", "group1_consumer1-0") }; Assert.Equal(expected_1, actual_1); // commit consumer offsets zkConsumerConnector1.CommitOffsets(); // create a consumer var consumerConfig2 = TestUtils.CreateConsumerProperties(ZkConnect, Group, Consumer2); consumerConfig2.RebalanceBackoffMs = RebalanceBackoutMs; var zkConsumerConnector2 = new ZookeeperConsumerConnector(consumerConfig2); var topicMessageStreams2 = zkConsumerConnector2.CreateMessageStreams( new Dictionary<string, int> { { Topic, 1 } }, new StringDecoder(), new StringDecoder()); // send some messages to each broker var sentMessages2 = this.SendMessagesToBrokerPartition(Configs.First(), Topic, 0, nMessages) .Union(this.SendMessagesToBrokerPartition(Configs.Last(), Topic, 1, nMessages)).ToList(); // wait to make sure the topic and partition have a leader for the successful case TestUtils.WaitUntilLeaderIsElectedOrChanged(this.ZkClient, Topic, 0, 500); TestUtils.WaitUntilLeaderIsElectedOrChanged(this.ZkClient, Topic, 1, 500); var receivedMessages2 = this.GetMessages(nMessages, topicMessageStreams1) .Union(this.GetMessages(nMessages, topicMessageStreams2)) .ToList(); Assert.Equal(sentMessages2.OrderBy(x => x).ToList(), receivedMessages2.OrderBy(x => x).ToList()); // also check partition ownership var actual_2 = this.GetZKChildrenValues(this.dirs.ConsumerOwnerDir); var expected_2 = new List<Tuple<string, string>> { Tuple.Create("0", "group1_consumer1-0"), Tuple.Create("1", "group1_consumer2-0") }; Assert.Equal(expected_2, actual_2); // create a consumer with empty map var consumerConfig3 = TestUtils.CreateConsumerProperties(ZkConnect, Group, Consumer3); var zkConsumerConnector3 = new ZookeeperConsumerConnector(consumerConfig3); zkConsumerConnector3.CreateMessageStreams(new Dictionary<string, int>()); // send some messages to each broker var sentMessages3 = this.SendMessagesToBrokerPartition(Configs.First(), Topic, 0, nMessages) .Union(this.SendMessagesToBrokerPartition(Configs.Last(), Topic, 1, nMessages)) .ToList(); // wait to make sure the topic and partition have a leader for the successful case TestUtils.WaitUntilLeaderIsElectedOrChanged(this.ZkClient, Topic, 0, 500); TestUtils.WaitUntilLeaderIsElectedOrChanged(this.ZkClient, Topic, 1, 500); var receivedMessages3 = this.GetMessages(nMessages, topicMessageStreams1) .Union(this.GetMessages(nMessages, topicMessageStreams2)) .ToList(); Assert.Equal(sentMessages3.OrderBy(x => x).ToList(), receivedMessages3.OrderBy(x => x).ToList()); // also check partition ownership var actual_3 = this.GetZKChildrenValues(this.dirs.ConsumerOwnerDir); Assert.Equal(expected_2, actual_3); zkConsumerConnector1.Shutdown(); zkConsumerConnector2.Shutdown(); zkConsumerConnector3.Shutdown(); Logger.Info("all consumer connectors stopped"); }
/// <summary> /// Create a ConsumerConnector /// </summary> /// <param name="config">at the minimum, need to specify the groupid of the consumer and the zookeeper connection string zookeeper.connect.</param> /// <returns></returns> public static IConsumerConnector Create(ConsumerConfig config) { var consumeConnect = new ZookeeperConsumerConnector(config); return consumeConnect; }
public void TestConsumerDecoder() { // send some messages to each broker var sentMessages = this.SendMessagesToBrokerPartition( Configs.First(), Topic, 0, nMessages) .Union( this.SendMessagesToBrokerPartition( Configs.First(), Topic, 1, nMessages)) .ToList(); TestUtils.WaitUntilMetadataIsPropagated(this.Servers, Topic, 0, 1000); TestUtils.WaitUntilMetadataIsPropagated(this.Servers, Topic, 1, 1000); // create a consuemr var consumerConfig = TestUtils.CreateConsumerProperties(ZkConnect, Group, Consumer1); TestUtils.WaitUntilLeaderIsElectedOrChanged(this.ZkClient, Topic, 0, 500); TestUtils.WaitUntilLeaderIsElectedOrChanged(this.ZkClient, Topic, 1, 500); var zkConsumerConnector = new ZookeeperConsumerConnector(consumerConfig); var topicMessageStreams = zkConsumerConnector.CreateMessageStreams( new Dictionary<string, int> { { Topic, 1 } }, new StringDecoder(), new StringDecoder()); var receivedMessages = this.GetMessages(nMessages * 2, topicMessageStreams); Assert.Equal(sentMessages.OrderBy(x => x).ToArray(), receivedMessages.OrderBy(x => x).ToArray()); zkConsumerConnector.Shutdown(); }
public void TestLeaderSelectionForPartition() { var zkClient = new ZkClient(this.zookeeperConnect, 6000, 30000, new ZkStringSerializer()); // create topic topic1 with 1 partition on broker 0 AdminUtils.CreateTopic(zkClient, Topic, 1, 1, new Dictionary<string, string>()); TestUtils.WaitUntilMetadataIsPropagated(this.Servers, Topic, 0, 3000); var sentMessages1 = this.SendMessages( Configs.First(), nMessages, "batch1", CompressionCodecs.NoCompressionCodec, 1); TestUtils.WaitUntilMetadataIsPropagated(this.Servers, Topic, 0, 1000); // create a consuemr var consumerConfig1 = TestUtils.CreateConsumerProperties(ZkConnect, Group, Consumer1); var zkConsumerConnector1 = new ZookeeperConsumerConnector(consumerConfig1); var topicMessageStreams1 = zkConsumerConnector1.CreateMessageStreams( new Dictionary<string, int> { { Topic, 1 } }, new StringDecoder(), new StringDecoder()); var topicRegistry = zkConsumerConnector1.TopicRegistry; Assert.Equal(1, topicRegistry.Select(x => x.Key).Count()); Assert.Equal(Topic, topicRegistry.Select(x => x.Key).First()); var topicsAndPartitionsInRegistry = topicRegistry.Select(x => Tuple.Create(x.Key, x.Value.Select(p => p.Value))).ToList(); var brokerPartition = topicsAndPartitionsInRegistry.First().Item2.First(); Assert.Equal(0, brokerPartition.PartitionId); // also check partition ownership var actual_1 = this.GetZKChildrenValues(this.dirs.ConsumerOwnerDir); var expected_1 = new List<Tuple<string, string>> { Tuple.Create("0", "group1_consumer1-0"), }; Assert.Equal(expected_1, actual_1); var receivedMessages1 = this.GetMessages(nMessages, topicMessageStreams1); Assert.Equal(sentMessages1, receivedMessages1); zkConsumerConnector1.Shutdown(); zkClient.Dispose(); }
internal void Consume() { // connects to zookeeper using (ZookeeperConsumerConnector connector = new ZookeeperConsumerConnector(configSettings, true)) { if (this.ThreadID == 0) { ConsumerGroupHelper.initialOffset = connector.GetOffset(cgOptions.Topic); Logger.InfoFormat("======Original offset \r\n{0}", ConsumerGroupHelper.initialOffset == null ? "(NULL)" : ConsumeGroupMonitorHelper.GetComsumerGroupOffsetsAsLog(ConsumerGroupHelper.initialOffset)); } // defines collection of topics and number of threads to consume it with // ===============NOTE============================ // For example , if there is 80 partitions for one topic. // // Normally start more than 96 = 80*120% clients with same GroupId. ( the extra 20% are buffer for autopilot IMP). And set FetchThreadCountPerConsumer as 1. // Then 80 clients can lock partitions, can set MACHINENAME_ProcessID as ConsumerId, other 16 are idle. Strongly recomand take this method. // // If start 40 clients, and set FetchThreadCountPerConsumer as 1. then every client can lock 2 partitions at least. And if some client not available for autopilot // IMP reason, then some of the client maybe lock 3 partitions. // // If start 40 clients, and set FetchThreadCountPerConsumer as 2, you will get two IEnumerator<Message>:topicData[0].GetEnumerator(),topicData[1].GetEnumerator() // you need start TWO threads to process them in dependently. // If the client get 2 partitions, each thread will handle 1 partition, // If the client get 3 partitions, then one thread get 2 partitions, the other one get 1 partition. It will make the situaiton complex and the consume of partition not balance. //==================NOTE============================= IDictionary<string, int> topicMap = new Dictionary<string, int> { { cgOptions.Topic, cgOptions.FetchThreadCountPerConsumer } }; // get references to topic streams. IDictionary<string, IList<KafkaMessageStream<Message>>> streams = connector.CreateMessageStreams(topicMap, new DefaultDecoder()); IList<KafkaMessageStream<Message>> topicData = streams[cgOptions.Topic]; long latestTotalCount = 0; bool hitEndAndCommited = false; if (cgOptions.CancellationTimeoutMs == KafkaNETExampleConstants.DefaultCancellationTimeoutMs) { // Get the message enumerator. IEnumerator<Message> messageEnumerator = topicData[0].GetEnumerator(); //TODO: the enumerator count equal with FetchThreadCountPerConsumer . For example, if that value is 5, then here should get 5 enumerator. //IF have 100 partitions, and only 20 consumers, need set this value to 5. and start 5 thread handle each one. // Add tuples until maximum receive message count is reached or no new messages read after consumer configured timeout. while (true) { bool noMoreMessage = false; try { messageEnumerator.MoveNext(); Message m = messageEnumerator.Current; latestTotalCount = Interlocked.Increment(ref ConsumerGroupHelper.totalCount); Logger.InfoFormat("Message {0} from Partition:{1}, Offset:{2}, key:{3}, value:{4}", latestTotalCount, m.PartitionId, m.Offset, m.Key == null ? "(null)" : Encoding.UTF8.GetString(m.Key), m.Payload == null ? "(null)" : Encoding.UTF8.GetString(m.Payload)); if (latestTotalCount == 1) { Logger.InfoFormat("Read FIRST message, it's offset: {0} PartitionID:{1}", m.Offset, ((ConsumerIterator<Message>)messageEnumerator).currentTopicInfo.PartitionId); } hitEndAndCommited = false; if (latestTotalCount % cgOptions.CommitBatchSize == 0) { //NOTE====== //Normally, just directly call .CommitOffsets() // CommitOffset(string topic, int partition, long offset) only used when customer has strong requirement for reprocess messages as few as possible. //Need tune the frequecy of calling .CommitOffsets(), it will directly increate zookeeper load and impact your overall performance if (cgOptions.CommitOffsetWithPartitionIDOffset) connector.CommitOffset(cgOptions.Topic, m.PartitionId.Value, m.Offset); else connector.CommitOffsets(); Console.WriteLine("\tRead some and commit once, LATEST message offset: {0}. PartitionID:{1} -- {2} Totally read {3} will commit offset. {4} FetchOffset:{5} ConsumeOffset:{6} CommitedOffset:{7}" , m.Offset, m.PartitionId.Value, ((ConsumerIterator<Message>)messageEnumerator).currentTopicInfo.PartitionId, latestTotalCount, DateTime.Now , ((ConsumerIterator<Message>)messageEnumerator).currentTopicInfo.FetchOffset , ((ConsumerIterator<Message>)messageEnumerator).currentTopicInfo.ConsumeOffset , ((ConsumerIterator<Message>)messageEnumerator).currentTopicInfo.CommitedOffset); } if (cgOptions.Count > 0 && latestTotalCount >= cgOptions.Count) { Logger.InfoFormat("Read LAST message, it's offset: {0}. PartitionID:{1} Totally read {2} want {3} will exit.", m.Offset, ((ConsumerIterator<Message>)messageEnumerator).currentTopicInfo.PartitionId, latestTotalCount, cgOptions.Count); break; } } catch (ConsumerTimeoutException) { if (!hitEndAndCommited) { Logger.InfoFormat("Totally Read {0} will commit offset. {1}", latestTotalCount, DateTime.Now); connector.CommitOffsets(); hitEndAndCommited = true; } // Thrown if no new messages read after consumer configured timeout. noMoreMessage = true; } if (noMoreMessage) { Logger.InfoFormat("No more message , hit end ,will Sleep(1), {0}", DateTime.Now); if (cgOptions.SleepTypeWhileAlwaysRead == 0) Thread.Sleep(0); else if (cgOptions.SleepTypeWhileAlwaysRead == 1) Thread.Sleep(1); //Best choice is Thread.Sleep(1). Other 3 choice still make the CPU 100% else if (cgOptions.SleepTypeWhileAlwaysRead == 2) Thread.Yield(); else { } } } } else { //Siphon scenario, repeatly take some messages and process. if no enough messages, will stop current batch after timeout. while (true) { #if NET45 bool noMoreMessage = false; Message lastMessage = null; int count = 0; KafkaMessageStream<Message> messagesStream = null; ConsumerIterator<Message> iterator = null; using (CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(cgOptions.CancellationTimeoutMs)) { lastMessage = null; IEnumerable<Message> messages = topicData[0].GetCancellable(cancellationTokenSource.Token); messagesStream = (KafkaMessageStream<Message>)messages; iterator = (ConsumerIterator<Message>)messagesStream.iterator; foreach (Message message in messages) { latestTotalCount = Interlocked.Increment(ref ConsumerGroupHelper.totalCount); lastMessage = message; if (latestTotalCount == 1) { PartitionTopicInfo p = iterator.currentTopicInfo; Logger.InfoFormat("Read FIRST message, it's offset: {0} PartitionID:{1}", lastMessage.Offset, p == null ? "null" : p.PartitionId.ToString()); } hitEndAndCommited = false; if (++count >= cgOptions.CommitBatchSize) { cancellationTokenSource.Cancel(); } } } if (count > 0) { connector.CommitOffsets(); consumedTotalCount += count; PartitionTopicInfo p = iterator.currentTopicInfo; Console.WriteLine("\tRead some and commit once, Thread: {8} consumedTotalCount:{9} Target:{10} LATEST message offset: {0}. PartitionID:{1} -- {2} Totally read {3} will commit offset. {4} FetchOffset:{5} ConsumeOffset:{6} CommitedOffset:{7}" , lastMessage.Offset, lastMessage.PartitionId.Value, p == null ? "null" : p.PartitionId.ToString(), latestTotalCount, DateTime.Now , p == null ? "null" : p.FetchOffset.ToString() , p == null ? "null" : p.ConsumeOffset.ToString() , p == null ? "null" : p.CommitedOffset.ToString() , this.ThreadID , this.consumedTotalCount , this.Count); } else { noMoreMessage = true; } if (this.Count > 0 && consumedTotalCount >= this.Count) { Logger.InfoFormat("Current thrad Read LAST message, Totally read {0} want {1} will exit current thread.", consumedTotalCount, this.Count); break; } if (noMoreMessage) { Logger.InfoFormat("No more message , hit end ,will Sleep(2000), {0}", DateTime.Now); if (cgOptions.SleepTypeWhileAlwaysRead == 0) Thread.Sleep(0); else if (cgOptions.SleepTypeWhileAlwaysRead == 1) Thread.Sleep(2000); //Best choice is Thread.Sleep(1). Other 3 choice still make the CPU 100% else if (cgOptions.SleepTypeWhileAlwaysRead == 2) Thread.Yield(); else { } } #endif #if NET4 throw new NotSupportedException("Please use .net45 to compile ."); #endif } } Logger.InfoFormat("Read {0} will commit offset. {1}", latestTotalCount, DateTime.Now); connector.CommitOffsets(); latestTotalCount = Interlocked.Read(ref ConsumerGroupHelper.totalCount); Logger.InfoFormat("Totally read {0} want {1} . ", latestTotalCount, cgOptions.Count); if (this.ThreadID == 0) { ConsumerGroupHelper.newOffset = connector.GetOffset(cgOptions.Topic); } } this.resetEvent.Set(); }
public void TestCompressionSetConsumption() { // send some messages to each broker var sentMessages = this.SendMessagesToBrokerPartition( Configs.First(), Topic, 0, 200, CompressionCodecs.DefaultCompressionCodec) .Union( this.SendMessagesToBrokerPartition( Configs.First(), Topic, 1, 200, CompressionCodecs.DefaultCompressionCodec)) .ToList(); TestUtils.WaitUntilMetadataIsPropagated(this.Servers, Topic, 0, 1000); TestUtils.WaitUntilMetadataIsPropagated(this.Servers, Topic, 1, 1000); // create a consuemr var consumerConfig1 = TestUtils.CreateConsumerProperties(ZkConnect, Group, Consumer0); var zkConsumerConnector1 = new ZookeeperConsumerConnector(consumerConfig1); var topicMessageStreams1 = zkConsumerConnector1.CreateMessageStreams( new Dictionary<string, int> { { Topic, 1 } }, new StringDecoder(), new StringDecoder()); var receivedMessages1 = this.GetMessages(400, topicMessageStreams1); Assert.Equal(sentMessages.OrderBy(x => x).ToArray(), receivedMessages1.OrderBy(x => x).ToArray()); // also check partition ownership var actual_2 = this.GetZKChildrenValues(this.dirs.ConsumerOwnerDir); var expected_2 = new List<Tuple<string, string>> { Tuple.Create("0", "group1_consumer0-0"), Tuple.Create("1", "group1_consumer0-0") }; Assert.Equal(expected_2, actual_2); zkConsumerConnector1.Shutdown(); }
public ZKTopicPartitionChangeListener( ZookeeperConsumerConnector parent, IZKRebalancerListener loadBalancerListener) { this.parent = parent; this.LoadbalancerListener = loadBalancerListener; }
public void ConsumerPorformsRebalancingWhenBrokerIsRemovedFromTopic() { var config = new ConsumerConfig(clientConfig) { AutoCommit = false, GroupId = "group1", ZkSessionTimeoutMs = 60000, ZkConnectionTimeoutMs = 60000 }; string brokerPath = ZooKeeperClient.DefaultBrokerIdsPath + "/" + 2345; string brokerTopicPath = ZooKeeperClient.DefaultBrokerTopicsPath + "/test/" + 2345; using (var consumerConnector = new ZookeeperConsumerConnector(config, true)) { var client = ReflectionHelper.GetInstanceField<ZooKeeperClient>("zkClient", consumerConnector); Assert.IsNotNull(client); client.DeleteRecursive("/consumers/group1"); var topicCount = new Dictionary<string, int> { { "test", 1 } }; consumerConnector.CreateMessageStreams(topicCount); WaitUntillIdle(client, 1000); client.CreateEphemeral(brokerPath, "192.168.1.39-1310449279123:192.168.1.39:9102"); client.CreateEphemeral(brokerTopicPath, 1); WaitUntillIdle(client, 1000); client.DeleteRecursive(brokerTopicPath); WaitUntillIdle(client, 1000); IList<string> children = client.GetChildren("/consumers/group1/owners/test", false); Assert.That(children.Count, Is.EqualTo(2)); Assert.That(children, Has.None.EqualTo("2345-0")); var topicRegistry = ReflectionHelper.GetInstanceField<IDictionary<string, IDictionary<Partition, PartitionTopicInfo>>>("topicRegistry", consumerConnector); Assert.That(topicRegistry, Is.Not.Null.And.Not.Empty); Assert.That(topicRegistry.Count, Is.EqualTo(1)); var item = topicRegistry["test"]; Assert.That(item.Count, Is.EqualTo(2)); Assert.That(item.Where(x => x.Value.BrokerId == 2345).Count(), Is.EqualTo(0)); } }
public void ConsumerConnectorReceivesAShutdownSignal() { // now consuming var config = new ConsumerConfig(clientConfig) { AutoCommit = false }; using (IConsumerConnector consumerConnector = new ZookeeperConsumerConnector(config, true)) { var topicCount = new Dictionary<string, int> { { CurrentTestTopic, 1 } }; var messages = consumerConnector.CreateMessageStreams(topicCount); // putting the shutdown command into the queue FieldInfo fi = typeof(ZookeeperConsumerConnector).GetField( "queues", BindingFlags.NonPublic | BindingFlags.Instance); var value = (IDictionary<Tuple<string, string>, BlockingCollection<FetchedDataChunk>>) fi.GetValue(consumerConnector); foreach (var topicConsumerQueueMap in value) { topicConsumerQueueMap.Value.Add(ZookeeperConsumerConnector.ShutdownCommand); } var sets = messages[CurrentTestTopic]; var resultMessages = new List<Message>(); foreach (var set in sets) { foreach (var message in set) { resultMessages.Add(message); } } Assert.AreEqual(0, resultMessages.Count); } }
public void ConsumerPorformsRebalancingWhenNewBrokerIsAddedToTopic() { var config = new ConsumerConfig(clientConfig) { AutoCommit = false, GroupId = "group1", ZkSessionTimeoutMs = 60000, ZkConnectionTimeoutMs = 60000 }; string brokerPath = ZooKeeperClient.DefaultBrokerIdsPath + "/" + 2345; string brokerTopicPath = ZooKeeperClient.DefaultBrokerTopicsPath + "/test/" + 2345; using (var consumerConnector = new ZookeeperConsumerConnector(config, true)) { var client = ReflectionHelper.GetInstanceField<ZooKeeperClient>( "zkClient", consumerConnector); Assert.IsNotNull(client); client.DeleteRecursive("/consumers/group1"); var topicCount = new Dictionary<string, int> { { "test", 1 } }; consumerConnector.CreateMessageStreams(topicCount); WaitUntillIdle(client, 1000); IList<string> children = client.GetChildren("/consumers/group1/ids", false); string consumerId = children[0]; client.CreateEphemeral(brokerPath, "192.168.1.39-1310449279123:192.168.1.39:9102"); client.CreateEphemeral(brokerTopicPath, 1); WaitUntillIdle(client, 500); children = client.GetChildren("/consumers/group1/owners/test", false); Assert.That(children.Count, Is.EqualTo(3)); Assert.That(children, Contains.Item("2345-0")); string data = client.ReadData<string>("/consumers/group1/owners/test/2345-0"); Assert.That(data, Is.Not.Null); Assert.That(data, Contains.Substring(consumerId)); var topicRegistry = ReflectionHelper.GetInstanceField<IDictionary<string, IDictionary<Partition, PartitionTopicInfo>>>( "topicRegistry", consumerConnector); Assert.That(topicRegistry, Is.Not.Null.And.Not.Empty); Assert.That(topicRegistry.Count, Is.EqualTo(1)); var item = topicRegistry["test"]; Assert.That(item.Count, Is.EqualTo(3)); var broker = topicRegistry["test"].SingleOrDefault(x => x.Key.BrokerId == 2345); Assert.That(broker, Is.Not.Null); } }
public void ConsumerPorformsRebalancingOnStart() { var config = new ConsumerConfig(clientConfig) { AutoCommit = false, GroupId = "group1" }; using (var consumerConnector = new ZookeeperConsumerConnector(config, true)) { ZooKeeperClient client = ReflectionHelper.GetInstanceField<ZooKeeperClient>("zkClient", consumerConnector); Assert.IsNotNull(client); client.DeleteRecursive("/consumers/group1"); var topicCount = new Dictionary<string, int> { { "test", 1 } }; consumerConnector.CreateMessageStreams(topicCount); WaitUntillIdle(client, 1000); IList<string> children = client.GetChildren("/consumers", false); Assert.That(children, Is.Not.Null.And.Not.Empty); Assert.That(children, Contains.Item("group1")); children = client.GetChildren("/consumers/group1", false); Assert.That(children, Is.Not.Null.And.Not.Empty); Assert.That(children, Contains.Item("ids")); Assert.That(children, Contains.Item("owners")); children = client.GetChildren("/consumers/group1/ids", false); Assert.That(children, Is.Not.Null.And.Not.Empty); string consumerId = children[0]; children = client.GetChildren("/consumers/group1/owners", false); Assert.That(children, Is.Not.Null.And.Not.Empty); Assert.That(children.Count, Is.EqualTo(1)); Assert.That(children, Contains.Item("test")); children = client.GetChildren("/consumers/group1/owners/test", false); Assert.That(children, Is.Not.Null.And.Not.Empty); Assert.That(children.Count, Is.EqualTo(2)); string partId = children[0]; string data = client.ReadData<string>("/consumers/group1/owners/test/" + partId); Assert.That(data, Is.Not.Null.And.Not.Empty); Assert.That(data, Contains.Substring(consumerId)); data = client.ReadData<string>("/consumers/group1/ids/" + consumerId); Assert.That(data, Is.Not.Null.And.Not.Empty); Assert.That(data, Is.EqualTo("{ \"test\": 1 }")); } using (var client = new ZooKeeperClient(config.ZkConnect, config.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer)) { client.Connect(); //// Should be created as ephemeral IList<string> children = client.GetChildren("/consumers/group1/ids"); Assert.That(children, Is.Null.Or.Empty); //// Should be created as ephemeral children = client.GetChildren("/consumers/group1/owners/test"); Assert.That(children, Is.Null.Or.Empty); } }
public void OneMessageIsSentAndReceivedThenExceptionsWhenNoMessageThenAnotherMessageIsSentAndReceived() { // first producing string payload1 = "kafka 1."; byte[] payloadData1 = Encoding.UTF8.GetBytes(payload1); var msg1 = new Message(payloadData1); var producerConfig = new SyncProducerConfig(clientConfig); var producer = new SyncProducer(producerConfig); var producerRequest = new ProducerRequest(CurrentTestTopic, 0, new List<Message> { msg1 }); producer.Send(producerRequest); // now consuming var config = new ConsumerConfig(clientConfig) { AutoCommit = false, Timeout = 5000 }; using (IConsumerConnector consumerConnector = new ZookeeperConsumerConnector(config, true)) { var topicCount = new Dictionary<string, int> { { CurrentTestTopic, 1 } }; var messages = consumerConnector.CreateMessageStreams(topicCount); var sets = messages[CurrentTestTopic]; KafkaMessageStream myStream = sets[0]; var enumerator = myStream.GetEnumerator(); Assert.IsTrue(enumerator.MoveNext()); Assert.AreEqual(msg1.ToString(), enumerator.Current.ToString()); Assert.Throws<ConsumerTimeoutException>(() => enumerator.MoveNext()); Assert.Throws<Exception>(() => enumerator.MoveNext()); // iterator is in failed state enumerator.Reset(); // producing again string payload2 = "kafka 2."; byte[] payloadData2 = Encoding.UTF8.GetBytes(payload2); var msg2 = new Message(payloadData2); var producerRequest2 = new ProducerRequest(CurrentTestTopic, 0, new List<Message> { msg2 }); producer.Send(producerRequest2); Thread.Sleep(3000); Assert.IsTrue(enumerator.MoveNext()); Assert.AreEqual(msg2.ToString(), enumerator.Current.ToString()); } }
/// <summary> /// Create a ConsumerConnector /// </summary> /// <param name="config">at the minimum, need to specify the groupid of the consumer and the zookeeper connection string zookeeper.connect.</param> /// <returns></returns> public static IConsumerConnector Create(ConsumerConfig config) { var consumeConnect = new ZookeeperConsumerConnector(config); return(consumeConnect); }
public void ConsumerConnectorConsumesTwoDifferentTopics() { string topic1 = CurrentTestTopic + "1"; string topic2 = CurrentTestTopic + "2"; // first producing string payload1 = "kafka 1."; byte[] payloadData1 = Encoding.UTF8.GetBytes(payload1); var msg1 = new Message(payloadData1); string payload2 = "kafka 2."; byte[] payloadData2 = Encoding.UTF8.GetBytes(payload2); var msg2 = new Message(payloadData2); var producerConfig = new SyncProducerConfig(clientConfig); var producer = new SyncProducer(producerConfig); var producerRequest1 = new ProducerRequest(topic1, 0, new List<Message> { msg1 }); producer.Send(producerRequest1); var producerRequest2 = new ProducerRequest(topic2, 0, new List<Message> { msg2 }); producer.Send(producerRequest2); // now consuming var config = new ConsumerConfig(clientConfig) { AutoCommit = false }; var resultMessages1 = new List<Message>(); var resultMessages2 = new List<Message>(); using (IConsumerConnector consumerConnector = new ZookeeperConsumerConnector(config, true)) { var topicCount = new Dictionary<string, int> { { topic1, 1 }, { topic2, 1 } }; var messages = consumerConnector.CreateMessageStreams(topicCount); Assert.IsTrue(messages.ContainsKey(topic1)); Assert.IsTrue(messages.ContainsKey(topic2)); var sets1 = messages[topic1]; try { foreach (var set in sets1) { foreach (var message in set) { resultMessages1.Add(message); } } } catch (ConsumerTimeoutException) { // do nothing, this is expected } var sets2 = messages[topic2]; try { foreach (var set in sets2) { foreach (var message in set) { resultMessages2.Add(message); } } } catch (ConsumerTimeoutException) { // do nothing, this is expected } } Assert.AreEqual(1, resultMessages1.Count); Assert.AreEqual(msg1.ToString(), resultMessages1[0].ToString()); Assert.AreEqual(1, resultMessages2.Count); Assert.AreEqual(msg2.ToString(), resultMessages2[0].ToString()); }
public void SimpleSyncProducerSends2MessagesAndConsumerConnectorGetsThemBack() { // first producing string payload1 = "kafka 1."; byte[] payloadData1 = Encoding.UTF8.GetBytes(payload1); var msg1 = new Message(payloadData1); string payload2 = "kafka 2."; byte[] payloadData2 = Encoding.UTF8.GetBytes(payload2); var msg2 = new Message(payloadData2); var producerConfig = new SyncProducerConfig(clientConfig); var producer = new SyncProducer(producerConfig); var producerRequest = new ProducerRequest(CurrentTestTopic, 0, new List<Message> { msg1, msg2 }); producer.Send(producerRequest); // now consuming var config = new ConsumerConfig(clientConfig) { AutoCommit = false }; var resultMessages = new List<Message>(); using (IConsumerConnector consumerConnector = new ZookeeperConsumerConnector(config, true)) { var topicCount = new Dictionary<string, int> { { CurrentTestTopic, 1 } }; var messages = consumerConnector.CreateMessageStreams(topicCount); var sets = messages[CurrentTestTopic]; try { foreach (var set in sets) { foreach (var message in set) { resultMessages.Add(message); } } } catch (ConsumerTimeoutException) { // do nothing, this is expected } } Assert.AreEqual(2, resultMessages.Count); Assert.AreEqual(msg1.ToString(), resultMessages[0].ToString()); Assert.AreEqual(msg2.ToString(), resultMessages[1].ToString()); }