internal void RegisterConsumerInZk(ZKGroupDirs dirs, string consumerIdString, TopicCount topicCount)
{
this.EnsuresNotDisposed();
Logger.InfoFormat(CultureInfo.CurrentCulture, "begin registering consumer {0} in ZK", consumerIdString);
ZkUtils.CreateEphemeralPathExpectConflict(this.zkClient, dirs.ConsumerRegistryDir + "/" + consumerIdString, topicCount.ToJsonString());
Logger.InfoFormat(CultureInfo.CurrentCulture, "end registering consumer {0} in ZK", consumerIdString);
}
private IDictionary <string, IList <KafkaStream <TKey, TValue> > > Consume <TKey, TValue>(IDictionary <string, int> topicCountMap, IDecoder <TKey> keyDecoder, IDecoder <TValue> valueDecoder)
{
Logger.Debug("entering consume");
if (topicCountMap == null)
{
throw new ArgumentNullException("topicCountMap");
}
var topicCount = TopicCount.ConstructTopicCount(consumerIdString, topicCountMap);
var topicThreadIds = topicCount.GetConsumerThreadIdsPerTopic();
// make a list of (queue,stream) pairs, one pair for each threadId
var queuesAndStreams = topicThreadIds.Values.SelectMany(threadIdSet => threadIdSet.Select(_ =>
{
var queue = new BlockingCollection <FetchedDataChunk>(this.Config.QueuedMaxMessages);
var stream = new KafkaStream <TKey, TValue>(
queue, this.Config.ConsumerTimeoutMs, keyDecoder, valueDecoder, this.Config.ClientId);
return(Tuple.Create(queue, stream));
})).ToList();
var dirs = new ZKGroupDirs(this.Config.GroupId);
this.RegisterConsumerInZK(dirs, consumerIdString, topicCount);
ReinitializeConsumer(topicCount, queuesAndStreams);
return((IDictionary <string, IList <KafkaStream <TKey, TValue> > >)loadBalancerListener.KafkaMessageAndMetadataStreams);
}
public ZKSessionExpireListener(ZKGroupDirs dirs, string consumerIdString, TopicCount topicCount, ZKRebalancerListener loadBalancerListener, ZookeeperConsumerConnector zkConsumerConnector)
{
this.consumerIdString = consumerIdString;
this.loadBalancerListener = loadBalancerListener;
this.zkConsumerConnector = zkConsumerConnector;
this.dirs = dirs;
this.topicCount = topicCount;
}
public ZKSessionExpireListener(ZookeeperConsumerConnector parent, ZKGroupDirs dirs, string consumerIdString, TopicCount topicCount, IZKRebalancerListener loadbalancerListener)
{
this.parent = parent;
this.Dirs = dirs;
this.ConsumerIdString = consumerIdString;
this.TopicCount = topicCount;
this.LoadbalancerListener = loadbalancerListener;
}
public static TopicCount ConstructTopicCount(string group, string consumerId, ZkClient zkClient)
{
var dirs = new ZKGroupDirs(group);
var topicCountString = ZkUtils.ReadData(zkClient, dirs.ConsumerRegistryDir + "/" + consumerId).Item1;
string subscriptionPattern;
IDictionary <string, int> topMap;
try
{
var parsedJson = JObject.Parse(topicCountString);
if (parsedJson != null)
{
var pattern = parsedJson.Get("pattern");
if (pattern != null)
{
subscriptionPattern = pattern.Value <string>();
}
else
{
throw new KafkaException("error constructing TopicCount:" + topicCountString);
}
var topMapObject = (IEnumerable <KeyValuePair <string, JToken> >)parsedJson.Get("subscription");
if (topMapObject != null)
{
topMap = topMapObject.ToDictionary(x => x.Key, x => x.Value.Value <int>());
}
else
{
throw new KafkaException("error constructing TopicCount:" + topicCountString);
}
}
else
{
throw new KafkaException("error constructing TopicCount:" + topicCountString);
}
}
catch (Exception e)
{
Logger.Error("error parsing consumer json string " + topicCountString, e);
throw;
}
var hasWhiteList = WhiteListPattern.Equals(subscriptionPattern);
var hasBlackList = BlackListPattern.Equals(subscriptionPattern);
if (topMap.Count == 0 || !(hasWhiteList || hasBlackList))
{
return(new StaticTopicCount(consumerId, topMap));
}
else
{
var regex = topMap.First().Key;
var numStreams = topMap.First().Value;
TopicFilter filter = hasWhiteList ? (TopicFilter) new Whitelist(regex) : new Blacklist(regex);
return(new WildcardTopicCount(zkClient, consumerId, filter, numStreams));
}
}
internal void RegisterConsumerInZk(ZKGroupDirs dirs, string consumerIdString, TopicCount topicCount)
{
this.EnsuresNotDisposed();
Logger.InfoFormat("begin registering consumer {0} in ZK", consumerIdString);
try
{
this.GetZkClient().SlimLock.EnterWriteLock();
ZkUtils.CreateEphemeralPathExpectConflict(this.GetZkClient(), dirs.ConsumerRegistryDir + "/" + consumerIdString, topicCount.ToJsonString());
Logger.InfoFormat("successfully registering consumer {0} in ZK", consumerIdString);
}
catch (Exception ex)
{
Logger.ErrorFormat("error in RegisterConsumerInZk CreateEphemeralPathExpectConflict : {0}", ex.FormatException());
}
finally
{
GetZkClient().SlimLock.ExitWriteLock();
}
}
internal ZKRebalancerListener(
ConsumerConfiguration config,
string consumerIdString,
IDictionary <string, IDictionary <Partition, PartitionTopicInfo> > topicRegistry,
IZooKeeperClient zkClient,
ZookeeperConsumerConnector zkConsumerConnector,
IDictionary <Tuple <string, string>, BlockingCollection <FetchedDataChunk> > queues,
Fetcher fetcher,
object syncLock)
{
this.syncLock = syncLock;
this.consumerIdString = consumerIdString;
this.config = config;
this.topicRegistry = topicRegistry;
this.zkClient = zkClient;
this.dirs = new ZKGroupDirs(config.GroupId);
this.zkConsumerConnector = zkConsumerConnector;
this.queues = queues;
this.fetcher = fetcher;
}
private void RegisterConsumerInZK(ZKGroupDirs dirs, string consumerIdString, TopicCount topicCount)
{
Logger.InfoFormat("begin registering consumer {0} in ZK", consumerIdString);
var timestamp = DateTimeHelper.CurrentTimeMilis();
var consumerRegistrationInfo = JsonConvert.SerializeObject(new
{
version = 1,
subscription = topicCount.TopicCountMap,
pattern = topicCount.Pattern,
timestamp
});
ZkUtils.CreateEphemeralPathExpectConflictHandleZKBug(
zkClient,
dirs.ConsumerRegistryDir + "/" + consumerIdString,
consumerRegistrationInfo,
null,
(consumerZKstring, consumer) => true,
this.Config.ZooKeeper.ZkSessionTimeoutMs);
Logger.InfoFormat("end registering consumer {0} in ZK", consumerIdString);
}
internal ZKRebalancerListener(
ConsumerConfiguration config,
string consumerIdString,
IDictionary <string, IDictionary <int, PartitionTopicInfo> > topicRegistry,
IZooKeeperClient zkClient,
ZookeeperConsumerConnector zkConsumerConnector,
IDictionary <Tuple <string, string>, BlockingCollection <FetchedDataChunk> > queues,
Fetcher fetcher,
IDictionary <string, IList <KafkaMessageStream <TData> > > kafkaMessageStreams,
TopicCount topicCount)
{
this.consumerIdString = consumerIdString;
this.config = config;
this.topicRegistry = topicRegistry;
this.zkClient = zkClient;
dirs = new ZKGroupDirs(config.GroupId);
this.zkConsumerConnector = zkConsumerConnector;
this.queues = queues;
this.fetcher = fetcher;
this.kafkaMessageStreams = kafkaMessageStreams;
this.topicCount = topicCount;
}
private void DeleteConsumerIdNode()
{
string consumerIdString = GetConsumerIdString();
ZKGroupDirs dirs = new ZKGroupDirs(this.config.GroupId);
string idsPath = dirs.ConsumerRegistryDir + "/" + consumerIdString;
Logger.InfoFormat("Will delete {0} in zookeeper due to zookeeperConsumerConnector dispose.", idsPath);
try
{
GetZkClient().SlimLock.EnterWriteLock();
ZkUtils.DeletePath(GetZkClient(), idsPath);
Logger.InfoFormat("Path {0} deleted succsessfully.", idsPath);
}
catch (Exception ex)
{
Logger.ErrorFormat("Path {0} FAILED to be deleted: {1}", idsPath, ex.FormatException());
}
finally
{
GetZkClient().SlimLock.ExitWriteLock();
}
}
private IDictionary <string, IList <KafkaMessageStream> > Consume(IDictionary <string, int> topicCountDict)
{
Logger.Debug("entering consume");
if (topicCountDict == null)
{
throw new ArgumentNullException();
}
var dirs = new ZKGroupDirs(this.config.GroupId);
var result = new Dictionary <string, IList <KafkaMessageStream> >();
var guid = Guid.NewGuid().ToString().Replace("-", string.Empty).Substring(0, 8);
string consumerUuid = string.Format("{0}-{1}-{2}", Dns.GetHostName(), DateTime.Now.Ticks, guid);
string consumerIdString = this.config.GroupId + "_" + consumerUuid;
var topicCount = new TopicCount(consumerIdString, topicCountDict);
// listener to consumer and partition changes
var loadBalancerListener = new ZKRebalancerListener(
this.config,
consumerIdString,
this.topicRegistry,
this.zkClient,
this,
queues,
this.fetcher,
this.syncLock,
result);
this.RegisterConsumerInZk(dirs, consumerIdString, topicCount);
this.zkClient.Subscribe(dirs.ConsumerRegistryDir, loadBalancerListener);
//// create a queue per topic per consumer thread
var consumerThreadIdsPerTopicMap = topicCount.GetConsumerThreadIdsPerTopic();
foreach (var topic in consumerThreadIdsPerTopicMap.Keys)
{
var streamList = new List <KafkaMessageStream>();
foreach (string threadId in consumerThreadIdsPerTopicMap[topic])
{
var stream = new BlockingCollection <FetchedDataChunk>(new ConcurrentQueue <FetchedDataChunk>(), config.MaxQueuedChunks);
this.queues.Add(new Tuple <string, string>(topic, threadId), stream);
streamList.Add(new KafkaMessageStream(stream, this.config.Timeout));
}
result.Add(topic, streamList);
Logger.DebugFormat(CultureInfo.CurrentCulture, "adding topic {0} and stream to map...", topic);
// register on broker partition path changes
string partitionPath = ZooKeeperClient.DefaultBrokerTopicsPath + "/" + topic;
this.zkClient.MakeSurePersistentPathExists(partitionPath);
this.zkClient.Subscribe(partitionPath, loadBalancerListener);
}
//// register listener for session expired event
this.zkClient.Subscribe(new ZKSessionExpireListener(dirs, consumerIdString, topicCount, loadBalancerListener, this));
//// explicitly trigger load balancing for this consumer););
lock (this.syncLock)
{
loadBalancerListener.SyncedRebalance();
}
return(result);
}
private void ReinitializeConsumer <TKey, TValue>(
TopicCount topicCount, IList <Tuple <BlockingCollection <FetchedDataChunk>, KafkaStream <TKey, TValue> > > queuesAndStreams)
{
var dirs = new ZKGroupDirs(this.Config.GroupId);
// listener to consumer and partition changes
if (loadBalancerListener == null)
{
var topicStreamsMaps = new Dictionary <string, IList <KafkaStream <TKey, TValue> > >();
loadBalancerListener = new ZKRebalancerListener <TKey, TValue>(this, this.Config.GroupId, consumerIdString, topicStreamsMaps);
}
// create listener for session expired event if not exist yet
if (sessionExpirationListener == null)
{
sessionExpirationListener = new ZKSessionExpireListener(this,
dirs, consumerIdString, topicCount, loadBalancerListener);
}
// create listener for topic partition change event if not exist yet
if (topicPartitionChangeListener == null)
{
topicPartitionChangeListener = new ZKTopicPartitionChangeListener(this, loadBalancerListener);
}
var topicStreamsMap = (IDictionary <string, IList <KafkaStream <TKey, TValue> > >)loadBalancerListener.KafkaMessageAndMetadataStreams;
// map of {topic -> Set(thread-1, thread-2, ...)}
var consumerThreadIdsPerTopic = topicCount.GetConsumerThreadIdsPerTopic();
IList <Tuple <BlockingCollection <FetchedDataChunk>, KafkaStream <TKey, TValue> > > allQueuesAndStreams = null;
if (topicCount is WildcardTopicCount)
{
/*
* Wild-card consumption streams share the same queues, so we need to
* duplicate the list for the subsequent zip operation.
*/
allQueuesAndStreams = Enumerable.Range(1, consumerThreadIdsPerTopic.Keys.Count).SelectMany(_ => queuesAndStreams).ToList();
}
else if (topicCount is StaticTopicCount)
{
allQueuesAndStreams = queuesAndStreams;
}
var topicThreadIds = consumerThreadIdsPerTopic.SelectMany(topicAndThreadIds =>
{
var topic = topicAndThreadIds.Key;
var threadIds = topicAndThreadIds.Value;
return(threadIds.Select(id => Tuple.Create(topic, id)));
}).ToList();
Contract.Assert(topicThreadIds.Count == allQueuesAndStreams.Count, string.Format("Mismatch betwen thread ID count ({0}) adn queue count ({1})", topicThreadIds.Count, allQueuesAndStreams.Count));
var threadQueueStreamPairs = topicThreadIds.Zip(allQueuesAndStreams, Tuple.Create).ToList();
foreach (var e in threadQueueStreamPairs)
{
var topicThreadId = e.Item1;
var q = e.Item2.Item1;
topicThreadIdAndQueues[topicThreadId] = q;
Logger.DebugFormat("Adding topicThreadId {0} and queue {1} to topicThreadIdAndQueues Data structure", topicThreadId, string.Join(",", q));
MetersFactory.NewGauge(this.Config.ClientId + "-" + this.Config.GroupId + "-" + topicThreadId.Item1 + "-" + topicThreadId.Item2 + "-FetchQueueSize", () => q.Count);
}
var groupedByTopic = threadQueueStreamPairs.GroupBy(x => x.Item1.Item1).ToList();
foreach (var e in groupedByTopic)
{
var topic = e.Key;
var streams = e.Select(x => x.Item2.Item2).ToList();
topicStreamsMap[topic] = streams;
Logger.DebugFormat("adding topic {0} and {1} stream to map", topic, streams.Count);
}
// listener to consumer and partition changes
zkClient.SubscribeStateChanges(sessionExpirationListener);
zkClient.SubscribeChildChanges(dirs.ConsumerRegistryDir, loadBalancerListener);
foreach (var topicAndSteams in topicStreamsMap)
{
// register on broker partition path changes
var topicPath = ZkUtils.BrokerTopicsPath + "/" + topicAndSteams.Key;
zkClient.SubscribeDataChanges(topicPath, topicPartitionChangeListener);
}
// explicitly trigger load balancing for this consumer
loadBalancerListener.SyncedRebalance();
}
private IDictionary <string, IList <KafkaMessageStream <TData> > > Consume <TData>(IDictionary <string, int> topicCountDict, IDecoder <TData> decoder)
{
Logger.Debug("entering consume");
if (topicCountDict == null)
{
throw new ArgumentNullException(nameof(topicCountDict));
}
var dirs = new ZKGroupDirs(this.config.GroupId);
var result = new Dictionary <string, IList <KafkaMessageStream <TData> > >();
string consumerIdString = GetConsumerIdString();
var topicCount = new TopicCount(consumerIdString, topicCountDict);
//// create a queue per topic per consumer thread
var consumerThreadIdsPerTopicMap = topicCount.GetConsumerThreadIdsPerTopic();
foreach (var topic in consumerThreadIdsPerTopicMap.Keys)
{
var streamList = new List <KafkaMessageStream <TData> >();
foreach (string threadId in consumerThreadIdsPerTopicMap[topic])
{
var stream = new BlockingCollection <FetchedDataChunk>(new ConcurrentQueue <FetchedDataChunk>());
this.queues.Add(new Tuple <string, string>(topic, threadId), stream);
streamList.Add(new KafkaMessageStream <TData>(topic, stream, this.config.Timeout, decoder));
}
result.Add(topic, streamList);
Logger.InfoFormat("adding topic {0} and stream to map...", topic);
}
// listener to consumer and partition changes
var loadBalancerListener = new ZKRebalancerListener <TData>(
this.config,
consumerIdString,
this.topicRegistry,
this.GetZkClient(),
this,
queues,
this.fetcher,
result,
topicCount);
if (this.consumerRebalanceHandler != null)
{
loadBalancerListener.ConsumerRebalance += this.consumerRebalanceHandler;
}
stopAsyncRebalancing.Add(loadBalancerListener.StopRebalance);
this.RegisterConsumerInZk(dirs, consumerIdString, topicCount);
//// register listener for session expired event
var zkSessionExpireListener = new ZKSessionExpireListener <TData>(dirs, consumerIdString, topicCount, loadBalancerListener, this);
if (this.zkSessionDisconnectedHandler != null)
{
zkSessionExpireListener.ZKSessionDisconnected += this.zkSessionDisconnectedHandler;
}
if (this.zkSessionExpiredHandler != null)
{
zkSessionExpireListener.ZKSessionExpired += this.zkSessionExpiredHandler;
}
this.GetZkClient().Subscribe(zkSessionExpireListener);
this.subscribedZookeeperStateCollection.Add(zkSessionExpireListener);
this.GetZkClient().Subscribe(dirs.ConsumerRegistryDir, loadBalancerListener);
this.subscribedChildCollection.Add(new Tuple <string, IZooKeeperChildListener>(dirs.ConsumerRegistryDir, loadBalancerListener));
result.ForEach(topicAndStreams =>
{
// register on broker partition path changes
string partitionPath = ZooKeeperClient.DefaultBrokerTopicsPath + "/" + topicAndStreams.Key;
if (this.GetZkClient().Exists(partitionPath))
{
this.GetZkClient().Subscribe(partitionPath, loadBalancerListener);
this.subscribedChildCollection.Add(new Tuple <string, IZooKeeperChildListener>(partitionPath, loadBalancerListener));
// Create a mapping of all topic partitions and their current leaders
var topicsAndPartitions = ZkUtils.GetPartitionsForTopics(this.GetZkClient(), new[] { topicAndStreams.Key });
Dictionary <string, int> partitionLeaderMap = new Dictionary <string, int>();
foreach (var partitionId in topicsAndPartitions[topicAndStreams.Key])
{
// Find/parse current partition leader for this partition and add it
// to the mapping object
var partitionStatePath = partitionPath + "/partitions/" + partitionId + "/state";
this.GetZkClient().MakeSurePersistentPathExists(partitionStatePath);
int?partitionLeader = ZkUtils.GetLeaderForPartition(this.GetZkClient(), topicAndStreams.Key, int.Parse(partitionId));
partitionLeaderMap.Add(partitionStatePath, partitionLeader.GetValueOrDefault(-1));
}
// listen for changes on the state nodes for the partitions
// this will indicate when a leader switches, or the in sync replicas change
var leaderListener = new ZkPartitionLeaderListener <TData>(loadBalancerListener, partitionLeaderMap);
foreach (var partitionId in topicsAndPartitions[topicAndStreams.Key])
{
var partitionStatePath = partitionPath + "/partitions/" + partitionId + "/state";
this.GetZkClient().Subscribe(partitionStatePath, leaderListener);
this.subscribedZookeeperDataCollection.Add(new Tuple <string, IZooKeeperDataListener>(partitionStatePath, leaderListener));
}
}
else
{
Logger.WarnFormat("The topic path at {0}, does not exist.", partitionPath);
}
});
//// explicitly trigger load balancing for this consumer
Logger.Info("Performing rebalancing. A new consumer has been added to consumer group: " + dirs.ConsumerRegistryDir + ", consumer: " + consumerIdString);
Logger.InfoFormat("Subscribe count: subscribedChildCollection:{0} , subscribedZookeeperStateCollection:{1} subscribedZookeeperDataCollection:{2} "
, subscribedChildCollection.Count, subscribedZookeeperStateCollection.Count, subscribedZookeeperDataCollection.Count);
//// When a new consumer join, need wait for rebalance finish to make sure Fetcher thread started.
loadBalancerListener.AsyncRebalance(DefaultWaitTimeForInitialRebalanceInSeconds * 1000);
return(result);
}