public static IDictionary <string, List <string> > GetConsumersPerTopic(ZkClient zkClient, string group)
{
var dirs = new ZKGroupDirs(group);
var consumers = GetChildrenParentMayNotExist(zkClient, dirs.ConsumerRegistryDir);
var consumerPerTopicMap = new Dictionary <string, List <string> >();
foreach (var consumer in consumers)
{
var topicCount = TopicCount.ConstructTopicCount(group, consumer, zkClient);
foreach (var topicAndConsumer in topicCount.GetConsumerThreadIdsPerTopic())
{
var topic = topicAndConsumer.Key;
var consumerThreadIdSet = topicAndConsumer.Value;
foreach (var consumerThreadId in consumerThreadIdSet)
{
var curConsumers = consumerPerTopicMap.Get(topic);
if (curConsumers != null)
{
curConsumers.Add(consumerThreadId);
}
else
{
consumerPerTopicMap[topic] = new List <string> {
consumerThreadId
};
}
}
}
}
consumerPerTopicMap = consumerPerTopicMap.ToDictionary(x => x.Key, x => x.Value.OrderBy(y => y).ToList());
return(consumerPerTopicMap);
}
public ZKSessionExpireListener(ZKGroupDirs dirs, string consumerIdString, TopicCount topicCount, ZKRebalancerListener loadBalancerListener, ZookeeperConsumerConnector zkConsumerConnector)
{
this.consumerIdString = consumerIdString;
this.loadBalancerListener = loadBalancerListener;
this.zkConsumerConnector = zkConsumerConnector;
this.dirs = dirs;
this.topicCount = topicCount;
}
public static TopicCount ConstructTopicCount(string group, string consumerId, ZkClient zkClient)
{
var dirs = new ZKGroupDirs(group);
var topicCountString = ZkUtils.ReadData(zkClient, dirs.ConsumerRegistryDir + "/" + consumerId).Item1;
string subscriptionPattern;
IDictionary<string, int> topMap;
try
{
var parsedJson = JObject.Parse(topicCountString);
if (parsedJson != null)
{
var pattern = parsedJson.Get("pattern");
if (pattern != null)
{
subscriptionPattern = pattern.Value<string>();
}
else
{
throw new KafkaException("error constructing TopicCount:" + topicCountString);
}
var topMapObject = (IEnumerable<KeyValuePair<string, JToken>>)parsedJson.Get("subscription");
if (topMapObject != null)
{
topMap = topMapObject.ToDictionary(x => x.Key, x => x.Value.Value<int>());
}
else
{
throw new KafkaException("error constructing TopicCount:" + topicCountString);
}
}
else
{
throw new KafkaException("error constructing TopicCount:" + topicCountString);
}
}
catch (Exception e)
{
Logger.Error("error parsing consumer json string " + topicCountString, e);
throw;
}
var hasWhiteList = WhiteListPattern.Equals(subscriptionPattern);
var hasBlackList = BlackListPattern.Equals(subscriptionPattern);
if (topMap.Count == 0 || !(hasWhiteList || hasBlackList))
{
return new StaticTopicCount(consumerId, topMap);
}
else
{
var regex = topMap.First().Key;
var numStreams = topMap.First().Value;
TopicFilter filter = hasWhiteList ? (TopicFilter)new Whitelist(regex) : new Blacklist(regex);
return new WildcardTopicCount(zkClient, consumerId, filter, numStreams);
}
}
internal ZKRebalancerListener(
ConsumerConfiguration config,
string consumerIdString,
IDictionary<string, IDictionary<Partition, PartitionTopicInfo>> topicRegistry,
IZooKeeperClient zkClient,
ZookeeperConsumerConnector zkConsumerConnector,
IDictionary<Tuple<string, string>, BlockingCollection<FetchedDataChunk>> queues,
Fetcher fetcher,
object syncLock)
{
this.syncLock = syncLock;
this.consumerIdString = consumerIdString;
this.config = config;
this.topicRegistry = topicRegistry;
this.zkClient = zkClient;
this.dirs = new ZKGroupDirs(config.GroupId);
this.zkConsumerConnector = zkConsumerConnector;
this.queues = queues;
this.fetcher = fetcher;
}
private IDictionary<string, IList<KafkaMessageStream>> Consume(IDictionary<string, int> topicCountDict)
{
Logger.Debug("entering consume");
if (topicCountDict == null)
{
throw new ArgumentNullException();
}
var dirs = new ZKGroupDirs(this.config.GroupId);
var result = new Dictionary<string, IList<KafkaMessageStream>>();
string consumerUuid = Environment.MachineName + "-" + DateTime.Now.Millisecond;
string consumerIdString = this.config.GroupId + "_" + consumerUuid;
var topicCount = new TopicCount(consumerIdString, topicCountDict);
// listener to consumer and partition changes
var loadBalancerListener = new ZKRebalancerListener(
this.config,
consumerIdString,
this.topicRegistry,
this.zkClient,
this,
queues,
this.fetcher,
this.syncLock);
this.RegisterConsumerInZk(dirs, consumerIdString, topicCount);
this.zkClient.Subscribe(dirs.ConsumerRegistryDir, loadBalancerListener);
//// create a queue per topic per consumer thread
var consumerThreadIdsPerTopicMap = topicCount.GetConsumerThreadIdsPerTopic();
foreach (var topic in consumerThreadIdsPerTopicMap.Keys)
{
var streamList = new List<KafkaMessageStream>();
foreach (string threadId in consumerThreadIdsPerTopicMap[topic])
{
var stream = new BlockingCollection<FetchedDataChunk>(new ConcurrentQueue<FetchedDataChunk>());
this.queues.Add(new Tuple<string, string>(topic, threadId), stream);
streamList.Add(new KafkaMessageStream(stream, this.config.Timeout));
}
result.Add(topic, streamList);
Logger.DebugFormat(CultureInfo.CurrentCulture, "adding topic {0} and stream to map...", topic);
// register on broker partition path changes
string partitionPath = ZooKeeperClient.DefaultBrokerTopicsPath + "/" + topic;
this.zkClient.MakeSurePersistentPathExists(partitionPath);
this.zkClient.Subscribe(partitionPath, loadBalancerListener);
}
//// register listener for session expired event
this.zkClient.Subscribe(new ZKSessionExpireListener(dirs, consumerIdString, topicCount, loadBalancerListener, this));
//// explicitly trigger load balancing for this consumer
lock (this.syncLock)
{
loadBalancerListener.SyncedRebalance();
}
return result;
}
internal void RegisterConsumerInZk(ZKGroupDirs dirs, string consumerIdString, TopicCount topicCount)
{
this.EnsuresNotDisposed();
Logger.InfoFormat(CultureInfo.CurrentCulture, "begin registering consumer {0} in ZK", consumerIdString);
ZkUtils.CreateEphemeralPathExpectConflict(this.zkClient, dirs.ConsumerRegistryDir + "/" + consumerIdString, topicCount.ToJsonString());
Logger.InfoFormat(CultureInfo.CurrentCulture, "end registering consumer {0} in ZK", consumerIdString);
}
public static IDictionary<string, List<string>> GetConsumersPerTopic(ZkClient zkClient, string group)
{
var dirs = new ZKGroupDirs(group);
var consumers = GetChildrenParentMayNotExist(zkClient, dirs.ConsumerRegistryDir);
var consumerPerTopicMap = new Dictionary<string, List<string>>();
// Consumers are named by their group Id plus the machine name and an identifier suffix. In this context,
// we are asked to retrieve the consumers for the `group` argument, so we discard any group in /consumers/ids
// that is not prefixed with the group id (filter using Where Enumerable.Where extension method)
foreach (var consumer in consumers.Where(c => c.StartsWith(group)))
{
var topicCount = TopicCount.ConstructTopicCount(group, consumer, zkClient);
foreach (var topicAndConsumer in topicCount.GetConsumerThreadIdsPerTopic())
{
var topic = topicAndConsumer.Key;
var consumerThreadIdSet = topicAndConsumer.Value;
foreach (var consumerThreadId in consumerThreadIdSet)
{
var curConsumers = consumerPerTopicMap.Get(topic);
if (curConsumers != null)
{
curConsumers.Add(consumerThreadId);
}
else
{
consumerPerTopicMap[topic] = new List<string> { consumerThreadId };
}
}
}
}
consumerPerTopicMap = consumerPerTopicMap.ToDictionary(x => x.Key, x => x.Value.OrderBy(y => y).ToList());
return consumerPerTopicMap;
}
public static IDictionary<string, List<string>> GetConsumersPerTopic(ZkClient zkClient, string group)
{
var dirs = new ZKGroupDirs(group);
var consumers = GetChildrenParentMayNotExist(zkClient, dirs.ConsumerRegistryDir);
var consumerPerTopicMap = new Dictionary<string, List<string>>();
foreach (var consumer in consumers)
{
var topicCount = TopicCount.ConstructTopicCount(group, consumer, zkClient);
foreach (var topicAndConsumer in topicCount.GetConsumerThreadIdsPerTopic())
{
var topic = topicAndConsumer.Key;
var consumerThreadIdSet = topicAndConsumer.Value;
foreach (var consumerThreadId in consumerThreadIdSet)
{
var curConsumers = consumerPerTopicMap.Get(topic);
if (curConsumers != null)
{
curConsumers.Add(consumerThreadId);
}
else
{
consumerPerTopicMap[topic] = new List<string> { consumerThreadId };
}
}
}
}
consumerPerTopicMap = consumerPerTopicMap.ToDictionary(x => x.Key, x => x.Value.OrderBy(y => y).ToList());
return consumerPerTopicMap;
}