public static IDictionary <string, List <string> > GetConsumersPerTopic(ZkClient zkClient, string group) { var dirs = new ZKGroupDirs(group); var consumers = GetChildrenParentMayNotExist(zkClient, dirs.ConsumerRegistryDir); var consumerPerTopicMap = new Dictionary <string, List <string> >(); foreach (var consumer in consumers) { var topicCount = TopicCount.ConstructTopicCount(group, consumer, zkClient); foreach (var topicAndConsumer in topicCount.GetConsumerThreadIdsPerTopic()) { var topic = topicAndConsumer.Key; var consumerThreadIdSet = topicAndConsumer.Value; foreach (var consumerThreadId in consumerThreadIdSet) { var curConsumers = consumerPerTopicMap.Get(topic); if (curConsumers != null) { curConsumers.Add(consumerThreadId); } else { consumerPerTopicMap[topic] = new List <string> { consumerThreadId }; } } } } consumerPerTopicMap = consumerPerTopicMap.ToDictionary(x => x.Key, x => x.Value.OrderBy(y => y).ToList()); return(consumerPerTopicMap); }
public ZKSessionExpireListener(ZKGroupDirs dirs, string consumerIdString, TopicCount topicCount, ZKRebalancerListener loadBalancerListener, ZookeeperConsumerConnector zkConsumerConnector) { this.consumerIdString = consumerIdString; this.loadBalancerListener = loadBalancerListener; this.zkConsumerConnector = zkConsumerConnector; this.dirs = dirs; this.topicCount = topicCount; }
public static TopicCount ConstructTopicCount(string group, string consumerId, ZkClient zkClient) { var dirs = new ZKGroupDirs(group); var topicCountString = ZkUtils.ReadData(zkClient, dirs.ConsumerRegistryDir + "/" + consumerId).Item1; string subscriptionPattern; IDictionary<string, int> topMap; try { var parsedJson = JObject.Parse(topicCountString); if (parsedJson != null) { var pattern = parsedJson.Get("pattern"); if (pattern != null) { subscriptionPattern = pattern.Value<string>(); } else { throw new KafkaException("error constructing TopicCount:" + topicCountString); } var topMapObject = (IEnumerable<KeyValuePair<string, JToken>>)parsedJson.Get("subscription"); if (topMapObject != null) { topMap = topMapObject.ToDictionary(x => x.Key, x => x.Value.Value<int>()); } else { throw new KafkaException("error constructing TopicCount:" + topicCountString); } } else { throw new KafkaException("error constructing TopicCount:" + topicCountString); } } catch (Exception e) { Logger.Error("error parsing consumer json string " + topicCountString, e); throw; } var hasWhiteList = WhiteListPattern.Equals(subscriptionPattern); var hasBlackList = BlackListPattern.Equals(subscriptionPattern); if (topMap.Count == 0 || !(hasWhiteList || hasBlackList)) { return new StaticTopicCount(consumerId, topMap); } else { var regex = topMap.First().Key; var numStreams = topMap.First().Value; TopicFilter filter = hasWhiteList ? (TopicFilter)new Whitelist(regex) : new Blacklist(regex); return new WildcardTopicCount(zkClient, consumerId, filter, numStreams); } }
internal ZKRebalancerListener( ConsumerConfiguration config, string consumerIdString, IDictionary<string, IDictionary<Partition, PartitionTopicInfo>> topicRegistry, IZooKeeperClient zkClient, ZookeeperConsumerConnector zkConsumerConnector, IDictionary<Tuple<string, string>, BlockingCollection<FetchedDataChunk>> queues, Fetcher fetcher, object syncLock) { this.syncLock = syncLock; this.consumerIdString = consumerIdString; this.config = config; this.topicRegistry = topicRegistry; this.zkClient = zkClient; this.dirs = new ZKGroupDirs(config.GroupId); this.zkConsumerConnector = zkConsumerConnector; this.queues = queues; this.fetcher = fetcher; }
private IDictionary<string, IList<KafkaMessageStream>> Consume(IDictionary<string, int> topicCountDict) { Logger.Debug("entering consume"); if (topicCountDict == null) { throw new ArgumentNullException(); } var dirs = new ZKGroupDirs(this.config.GroupId); var result = new Dictionary<string, IList<KafkaMessageStream>>(); string consumerUuid = Environment.MachineName + "-" + DateTime.Now.Millisecond; string consumerIdString = this.config.GroupId + "_" + consumerUuid; var topicCount = new TopicCount(consumerIdString, topicCountDict); // listener to consumer and partition changes var loadBalancerListener = new ZKRebalancerListener( this.config, consumerIdString, this.topicRegistry, this.zkClient, this, queues, this.fetcher, this.syncLock); this.RegisterConsumerInZk(dirs, consumerIdString, topicCount); this.zkClient.Subscribe(dirs.ConsumerRegistryDir, loadBalancerListener); //// create a queue per topic per consumer thread var consumerThreadIdsPerTopicMap = topicCount.GetConsumerThreadIdsPerTopic(); foreach (var topic in consumerThreadIdsPerTopicMap.Keys) { var streamList = new List<KafkaMessageStream>(); foreach (string threadId in consumerThreadIdsPerTopicMap[topic]) { var stream = new BlockingCollection<FetchedDataChunk>(new ConcurrentQueue<FetchedDataChunk>()); this.queues.Add(new Tuple<string, string>(topic, threadId), stream); streamList.Add(new KafkaMessageStream(stream, this.config.Timeout)); } result.Add(topic, streamList); Logger.DebugFormat(CultureInfo.CurrentCulture, "adding topic {0} and stream to map...", topic); // register on broker partition path changes string partitionPath = ZooKeeperClient.DefaultBrokerTopicsPath + "/" + topic; this.zkClient.MakeSurePersistentPathExists(partitionPath); this.zkClient.Subscribe(partitionPath, loadBalancerListener); } //// register listener for session expired event this.zkClient.Subscribe(new ZKSessionExpireListener(dirs, consumerIdString, topicCount, loadBalancerListener, this)); //// explicitly trigger load balancing for this consumer lock (this.syncLock) { loadBalancerListener.SyncedRebalance(); } return result; }
internal void RegisterConsumerInZk(ZKGroupDirs dirs, string consumerIdString, TopicCount topicCount) { this.EnsuresNotDisposed(); Logger.InfoFormat(CultureInfo.CurrentCulture, "begin registering consumer {0} in ZK", consumerIdString); ZkUtils.CreateEphemeralPathExpectConflict(this.zkClient, dirs.ConsumerRegistryDir + "/" + consumerIdString, topicCount.ToJsonString()); Logger.InfoFormat(CultureInfo.CurrentCulture, "end registering consumer {0} in ZK", consumerIdString); }
public static IDictionary<string, List<string>> GetConsumersPerTopic(ZkClient zkClient, string group) { var dirs = new ZKGroupDirs(group); var consumers = GetChildrenParentMayNotExist(zkClient, dirs.ConsumerRegistryDir); var consumerPerTopicMap = new Dictionary<string, List<string>>(); // Consumers are named by their group Id plus the machine name and an identifier suffix. In this context, // we are asked to retrieve the consumers for the `group` argument, so we discard any group in /consumers/ids // that is not prefixed with the group id (filter using Where Enumerable.Where extension method) foreach (var consumer in consumers.Where(c => c.StartsWith(group))) { var topicCount = TopicCount.ConstructTopicCount(group, consumer, zkClient); foreach (var topicAndConsumer in topicCount.GetConsumerThreadIdsPerTopic()) { var topic = topicAndConsumer.Key; var consumerThreadIdSet = topicAndConsumer.Value; foreach (var consumerThreadId in consumerThreadIdSet) { var curConsumers = consumerPerTopicMap.Get(topic); if (curConsumers != null) { curConsumers.Add(consumerThreadId); } else { consumerPerTopicMap[topic] = new List<string> { consumerThreadId }; } } } } consumerPerTopicMap = consumerPerTopicMap.ToDictionary(x => x.Key, x => x.Value.OrderBy(y => y).ToList()); return consumerPerTopicMap; }
public static IDictionary<string, List<string>> GetConsumersPerTopic(ZkClient zkClient, string group) { var dirs = new ZKGroupDirs(group); var consumers = GetChildrenParentMayNotExist(zkClient, dirs.ConsumerRegistryDir); var consumerPerTopicMap = new Dictionary<string, List<string>>(); foreach (var consumer in consumers) { var topicCount = TopicCount.ConstructTopicCount(group, consumer, zkClient); foreach (var topicAndConsumer in topicCount.GetConsumerThreadIdsPerTopic()) { var topic = topicAndConsumer.Key; var consumerThreadIdSet = topicAndConsumer.Value; foreach (var consumerThreadId in consumerThreadIdSet) { var curConsumers = consumerPerTopicMap.Get(topic); if (curConsumers != null) { curConsumers.Add(consumerThreadId); } else { consumerPerTopicMap[topic] = new List<string> { consumerThreadId }; } } } } consumerPerTopicMap = consumerPerTopicMap.ToDictionary(x => x.Key, x => x.Value.OrderBy(y => y).ToList()); return consumerPerTopicMap; }