public ZKSessionExpireListener(ZKGroupDirs dirs, string consumerIdString, TopicCount topicCount, ZKRebalancerListener loadBalancerListener, ZookeeperConsumerConnector zkConsumerConnector) { this.consumerIdString = consumerIdString; this.loadBalancerListener = loadBalancerListener; this.zkConsumerConnector = zkConsumerConnector; this.dirs = dirs; this.topicCount = topicCount; }
public ZkPartitionLeaderListener(ZKRebalancerListener <TData> rebalancer, Dictionary <string, int> partitionLeaderMap = null) { _rebalancer = rebalancer; if (partitionLeaderMap != null) { _partitionLeaderMap = new Dictionary <string, int>(partitionLeaderMap); } else { _partitionLeaderMap = new Dictionary <string, int>(); } }
private IDictionary<string, IList<KafkaMessageStream>> Consume(IDictionary<string, int> topicCountDict) { Logger.Debug("entering consume"); if (topicCountDict == null) { throw new ArgumentNullException(); } var dirs = new ZKGroupDirs(this.config.GroupId); var result = new Dictionary<string, IList<KafkaMessageStream>>(); string consumerUuid = Environment.MachineName + "-" + DateTime.Now.Millisecond; string consumerIdString = this.config.GroupId + "_" + consumerUuid; var topicCount = new TopicCount(consumerIdString, topicCountDict); // listener to consumer and partition changes var loadBalancerListener = new ZKRebalancerListener( this.config, consumerIdString, this.topicRegistry, this.zkClient, this, queues, this.fetcher, this.syncLock); this.RegisterConsumerInZk(dirs, consumerIdString, topicCount); this.zkClient.Subscribe(dirs.ConsumerRegistryDir, loadBalancerListener); //// create a queue per topic per consumer thread var consumerThreadIdsPerTopicMap = topicCount.GetConsumerThreadIdsPerTopic(); foreach (var topic in consumerThreadIdsPerTopicMap.Keys) { var streamList = new List<KafkaMessageStream>(); foreach (string threadId in consumerThreadIdsPerTopicMap[topic]) { var stream = new BlockingCollection<FetchedDataChunk>(new ConcurrentQueue<FetchedDataChunk>()); this.queues.Add(new Tuple<string, string>(topic, threadId), stream); streamList.Add(new KafkaMessageStream(stream, this.config.Timeout)); } result.Add(topic, streamList); Logger.DebugFormat(CultureInfo.CurrentCulture, "adding topic {0} and stream to map...", topic); // register on broker partition path changes string partitionPath = ZooKeeperClient.DefaultBrokerTopicsPath + "/" + topic; this.zkClient.MakeSurePersistentPathExists(partitionPath); this.zkClient.Subscribe(partitionPath, loadBalancerListener); } //// register listener for session expired event this.zkClient.Subscribe(new ZKSessionExpireListener(dirs, consumerIdString, topicCount, loadBalancerListener, this)); //// explicitly trigger load balancing for this consumer lock (this.syncLock) { loadBalancerListener.SyncedRebalance(); } return result; }