private IDictionary <string, IList <KafkaMessageStream> > Consume(IDictionary <string, int> topicCountDict) { Logger.Debug("entering consume"); if (topicCountDict == null) { throw new ArgumentNullException(); } var dirs = new ZKGroupDirs(this.config.GroupId); var result = new Dictionary <string, IList <KafkaMessageStream> >(); var guid = Guid.NewGuid().ToString().Replace("-", string.Empty).Substring(0, 8); string consumerUuid = string.Format("{0}-{1}-{2}", Dns.GetHostName(), DateTime.Now.Ticks, guid); string consumerIdString = this.config.GroupId + "_" + consumerUuid; var topicCount = new TopicCount(consumerIdString, topicCountDict); // listener to consumer and partition changes var loadBalancerListener = new ZKRebalancerListener( this.config, consumerIdString, this.topicRegistry, this.zkClient, this, queues, this.fetcher, this.syncLock, result); this.RegisterConsumerInZk(dirs, consumerIdString, topicCount); this.zkClient.Subscribe(dirs.ConsumerRegistryDir, loadBalancerListener); //// create a queue per topic per consumer thread var consumerThreadIdsPerTopicMap = topicCount.GetConsumerThreadIdsPerTopic(); foreach (var topic in consumerThreadIdsPerTopicMap.Keys) { var streamList = new List <KafkaMessageStream>(); foreach (string threadId in consumerThreadIdsPerTopicMap[topic]) { var stream = new BlockingCollection <FetchedDataChunk>(new ConcurrentQueue <FetchedDataChunk>(), config.MaxQueuedChunks); this.queues.Add(new Tuple <string, string>(topic, threadId), stream); streamList.Add(new KafkaMessageStream(stream, this.config.Timeout)); } result.Add(topic, streamList); Logger.DebugFormat(CultureInfo.CurrentCulture, "adding topic {0} and stream to map...", topic); // register on broker partition path changes string partitionPath = ZooKeeperClient.DefaultBrokerTopicsPath + "/" + topic; this.zkClient.MakeSurePersistentPathExists(partitionPath); this.zkClient.Subscribe(partitionPath, loadBalancerListener); } //// register listener for session expired event this.zkClient.Subscribe(new ZKSessionExpireListener(dirs, consumerIdString, topicCount, loadBalancerListener, this)); //// explicitly trigger load balancing for this consumer);); lock (this.syncLock) { loadBalancerListener.SyncedRebalance(); } return(result); }
private IDictionary <string, IList <KafkaMessageStream <TData> > > Consume <TData>(IDictionary <string, int> topicCountDict, IDecoder <TData> decoder) { Logger.Debug("entering consume"); if (topicCountDict == null) { throw new ArgumentNullException(nameof(topicCountDict)); } var dirs = new ZKGroupDirs(this.config.GroupId); var result = new Dictionary <string, IList <KafkaMessageStream <TData> > >(); string consumerIdString = GetConsumerIdString(); var topicCount = new TopicCount(consumerIdString, topicCountDict); //// create a queue per topic per consumer thread var consumerThreadIdsPerTopicMap = topicCount.GetConsumerThreadIdsPerTopic(); foreach (var topic in consumerThreadIdsPerTopicMap.Keys) { var streamList = new List <KafkaMessageStream <TData> >(); foreach (string threadId in consumerThreadIdsPerTopicMap[topic]) { var stream = new BlockingCollection <FetchedDataChunk>(new ConcurrentQueue <FetchedDataChunk>()); this.queues.Add(new Tuple <string, string>(topic, threadId), stream); streamList.Add(new KafkaMessageStream <TData>(topic, stream, this.config.Timeout, decoder)); } result.Add(topic, streamList); Logger.InfoFormat("adding topic {0} and stream to map...", topic); } // listener to consumer and partition changes var loadBalancerListener = new ZKRebalancerListener <TData>( this.config, consumerIdString, this.topicRegistry, this.GetZkClient(), this, queues, this.fetcher, result, topicCount); if (this.consumerRebalanceHandler != null) { loadBalancerListener.ConsumerRebalance += this.consumerRebalanceHandler; } stopAsyncRebalancing.Add(loadBalancerListener.StopRebalance); this.RegisterConsumerInZk(dirs, consumerIdString, topicCount); //// register listener for session expired event var zkSessionExpireListener = new ZKSessionExpireListener <TData>(dirs, consumerIdString, topicCount, loadBalancerListener, this); if (this.zkSessionDisconnectedHandler != null) { zkSessionExpireListener.ZKSessionDisconnected += this.zkSessionDisconnectedHandler; } if (this.zkSessionExpiredHandler != null) { zkSessionExpireListener.ZKSessionExpired += this.zkSessionExpiredHandler; } this.GetZkClient().Subscribe(zkSessionExpireListener); this.subscribedZookeeperStateCollection.Add(zkSessionExpireListener); this.GetZkClient().Subscribe(dirs.ConsumerRegistryDir, loadBalancerListener); this.subscribedChildCollection.Add(new Tuple <string, IZooKeeperChildListener>(dirs.ConsumerRegistryDir, loadBalancerListener)); result.ForEach(topicAndStreams => { // register on broker partition path changes string partitionPath = ZooKeeperClient.DefaultBrokerTopicsPath + "/" + topicAndStreams.Key; if (this.GetZkClient().Exists(partitionPath)) { this.GetZkClient().Subscribe(partitionPath, loadBalancerListener); this.subscribedChildCollection.Add(new Tuple <string, IZooKeeperChildListener>(partitionPath, loadBalancerListener)); // Create a mapping of all topic partitions and their current leaders var topicsAndPartitions = ZkUtils.GetPartitionsForTopics(this.GetZkClient(), new[] { topicAndStreams.Key }); Dictionary <string, int> partitionLeaderMap = new Dictionary <string, int>(); foreach (var partitionId in topicsAndPartitions[topicAndStreams.Key]) { // Find/parse current partition leader for this partition and add it // to the mapping object var partitionStatePath = partitionPath + "/partitions/" + partitionId + "/state"; this.GetZkClient().MakeSurePersistentPathExists(partitionStatePath); int?partitionLeader = ZkUtils.GetLeaderForPartition(this.GetZkClient(), topicAndStreams.Key, int.Parse(partitionId)); partitionLeaderMap.Add(partitionStatePath, partitionLeader.GetValueOrDefault(-1)); } // listen for changes on the state nodes for the partitions // this will indicate when a leader switches, or the in sync replicas change var leaderListener = new ZkPartitionLeaderListener <TData>(loadBalancerListener, partitionLeaderMap); foreach (var partitionId in topicsAndPartitions[topicAndStreams.Key]) { var partitionStatePath = partitionPath + "/partitions/" + partitionId + "/state"; this.GetZkClient().Subscribe(partitionStatePath, leaderListener); this.subscribedZookeeperDataCollection.Add(new Tuple <string, IZooKeeperDataListener>(partitionStatePath, leaderListener)); } } else { Logger.WarnFormat("The topic path at {0}, does not exist.", partitionPath); } }); //// explicitly trigger load balancing for this consumer Logger.Info("Performing rebalancing. A new consumer has been added to consumer group: " + dirs.ConsumerRegistryDir + ", consumer: " + consumerIdString); Logger.InfoFormat("Subscribe count: subscribedChildCollection:{0} , subscribedZookeeperStateCollection:{1} subscribedZookeeperDataCollection:{2} " , subscribedChildCollection.Count, subscribedZookeeperStateCollection.Count, subscribedZookeeperDataCollection.Count); //// When a new consumer join, need wait for rebalance finish to make sure Fetcher thread started. loadBalancerListener.AsyncRebalance(DefaultWaitTimeForInitialRebalanceInSeconds * 1000); return(result); }