|
public GetConsumerThreadIdsPerTopic ( ) : IList |
||
| return | IList | |
private IDictionary <string, IList <KafkaMessageStream> > Consume(IDictionary <string, int> topicCountDict)
{
Logger.Debug("entering consume");
if (topicCountDict == null)
{
throw new ArgumentNullException();
}
var dirs = new ZKGroupDirs(this.config.GroupId);
var result = new Dictionary <string, IList <KafkaMessageStream> >();
var guid = Guid.NewGuid().ToString().Replace("-", string.Empty).Substring(0, 8);
string consumerUuid = string.Format("{0}-{1}-{2}", Dns.GetHostName(), DateTime.Now.Ticks, guid);
string consumerIdString = this.config.GroupId + "_" + consumerUuid;
var topicCount = new TopicCount(consumerIdString, topicCountDict);
// listener to consumer and partition changes
var loadBalancerListener = new ZKRebalancerListener(
this.config,
consumerIdString,
this.topicRegistry,
this.zkClient,
this,
queues,
this.fetcher,
this.syncLock,
result);
this.RegisterConsumerInZk(dirs, consumerIdString, topicCount);
this.zkClient.Subscribe(dirs.ConsumerRegistryDir, loadBalancerListener);
//// create a queue per topic per consumer thread
var consumerThreadIdsPerTopicMap = topicCount.GetConsumerThreadIdsPerTopic();
foreach (var topic in consumerThreadIdsPerTopicMap.Keys)
{
var streamList = new List <KafkaMessageStream>();
foreach (string threadId in consumerThreadIdsPerTopicMap[topic])
{
var stream = new BlockingCollection <FetchedDataChunk>(new ConcurrentQueue <FetchedDataChunk>(), config.MaxQueuedChunks);
this.queues.Add(new Tuple <string, string>(topic, threadId), stream);
streamList.Add(new KafkaMessageStream(stream, this.config.Timeout));
}
result.Add(topic, streamList);
Logger.DebugFormat(CultureInfo.CurrentCulture, "adding topic {0} and stream to map...", topic);
// register on broker partition path changes
string partitionPath = ZooKeeperClient.DefaultBrokerTopicsPath + "/" + topic;
this.zkClient.MakeSurePersistentPathExists(partitionPath);
this.zkClient.Subscribe(partitionPath, loadBalancerListener);
}
//// register listener for session expired event
this.zkClient.Subscribe(new ZKSessionExpireListener(dirs, consumerIdString, topicCount, loadBalancerListener, this));
//// explicitly trigger load balancing for this consumer););
lock (this.syncLock)
{
loadBalancerListener.SyncedRebalance();
}
return(result);
}
private void ReinitializeConsumer <TKey, TValue>(
TopicCount topicCount, IList <Tuple <BlockingCollection <FetchedDataChunk>, KafkaStream <TKey, TValue> > > queuesAndStreams)
{
var dirs = new ZKGroupDirs(this.Config.GroupId);
// listener to consumer and partition changes
if (loadBalancerListener == null)
{
var topicStreamsMaps = new Dictionary <string, IList <KafkaStream <TKey, TValue> > >();
loadBalancerListener = new ZKRebalancerListener <TKey, TValue>(this, this.Config.GroupId, consumerIdString, topicStreamsMaps);
}
// create listener for session expired event if not exist yet
if (sessionExpirationListener == null)
{
sessionExpirationListener = new ZKSessionExpireListener(this,
dirs, consumerIdString, topicCount, loadBalancerListener);
}
// create listener for topic partition change event if not exist yet
if (topicPartitionChangeListener == null)
{
topicPartitionChangeListener = new ZKTopicPartitionChangeListener(this, loadBalancerListener);
}
var topicStreamsMap = (IDictionary <string, IList <KafkaStream <TKey, TValue> > >)loadBalancerListener.KafkaMessageAndMetadataStreams;
// map of {topic -> Set(thread-1, thread-2, ...)}
var consumerThreadIdsPerTopic = topicCount.GetConsumerThreadIdsPerTopic();
IList <Tuple <BlockingCollection <FetchedDataChunk>, KafkaStream <TKey, TValue> > > allQueuesAndStreams = null;
if (topicCount is WildcardTopicCount)
{
/*
* Wild-card consumption streams share the same queues, so we need to
* duplicate the list for the subsequent zip operation.
*/
allQueuesAndStreams = Enumerable.Range(1, consumerThreadIdsPerTopic.Keys.Count).SelectMany(_ => queuesAndStreams).ToList();
}
else if (topicCount is StaticTopicCount)
{
allQueuesAndStreams = queuesAndStreams;
}
var topicThreadIds = consumerThreadIdsPerTopic.SelectMany(topicAndThreadIds =>
{
var topic = topicAndThreadIds.Key;
var threadIds = topicAndThreadIds.Value;
return(threadIds.Select(id => Tuple.Create(topic, id)));
}).ToList();
Contract.Assert(topicThreadIds.Count == allQueuesAndStreams.Count, string.Format("Mismatch betwen thread ID count ({0}) adn queue count ({1})", topicThreadIds.Count, allQueuesAndStreams.Count));
var threadQueueStreamPairs = topicThreadIds.Zip(allQueuesAndStreams, Tuple.Create).ToList();
foreach (var e in threadQueueStreamPairs)
{
var topicThreadId = e.Item1;
var q = e.Item2.Item1;
topicThreadIdAndQueues[topicThreadId] = q;
Logger.DebugFormat("Adding topicThreadId {0} and queue {1} to topicThreadIdAndQueues Data structure", topicThreadId, string.Join(",", q));
MetersFactory.NewGauge(this.Config.ClientId + "-" + this.Config.GroupId + "-" + topicThreadId.Item1 + "-" + topicThreadId.Item2 + "-FetchQueueSize", () => q.Count);
}
var groupedByTopic = threadQueueStreamPairs.GroupBy(x => x.Item1.Item1).ToList();
foreach (var e in groupedByTopic)
{
var topic = e.Key;
var streams = e.Select(x => x.Item2.Item2).ToList();
topicStreamsMap[topic] = streams;
Logger.DebugFormat("adding topic {0} and {1} stream to map", topic, streams.Count);
}
// listener to consumer and partition changes
zkClient.SubscribeStateChanges(sessionExpirationListener);
zkClient.SubscribeChildChanges(dirs.ConsumerRegistryDir, loadBalancerListener);
foreach (var topicAndSteams in topicStreamsMap)
{
// register on broker partition path changes
var topicPath = ZkUtils.BrokerTopicsPath + "/" + topicAndSteams.Key;
zkClient.SubscribeDataChanges(topicPath, topicPartitionChangeListener);
}
// explicitly trigger load balancing for this consumer
loadBalancerListener.SyncedRebalance();
}
private IDictionary<string, IList<KafkaMessageStream>> Consume(IDictionary<string, int> topicCountDict)
{
Logger.Debug("entering consume");
if (topicCountDict == null)
{
throw new ArgumentNullException();
}
var dirs = new ZKGroupDirs(this.config.GroupId);
var result = new Dictionary<string, IList<KafkaMessageStream>>();
string consumerUuid = Environment.MachineName + "-" + DateTime.Now.Millisecond;
string consumerIdString = this.config.GroupId + "_" + consumerUuid;
var topicCount = new TopicCount(consumerIdString, topicCountDict);
// listener to consumer and partition changes
var loadBalancerListener = new ZKRebalancerListener(
this.config,
consumerIdString,
this.topicRegistry,
this.zkClient,
this,
queues,
this.fetcher,
this.syncLock);
this.RegisterConsumerInZk(dirs, consumerIdString, topicCount);
this.zkClient.Subscribe(dirs.ConsumerRegistryDir, loadBalancerListener);
//// create a queue per topic per consumer thread
var consumerThreadIdsPerTopicMap = topicCount.GetConsumerThreadIdsPerTopic();
foreach (var topic in consumerThreadIdsPerTopicMap.Keys)
{
var streamList = new List<KafkaMessageStream>();
foreach (string threadId in consumerThreadIdsPerTopicMap[topic])
{
var stream = new BlockingCollection<FetchedDataChunk>(new ConcurrentQueue<FetchedDataChunk>());
this.queues.Add(new Tuple<string, string>(topic, threadId), stream);
streamList.Add(new KafkaMessageStream(stream, this.config.Timeout));
}
result.Add(topic, streamList);
Logger.DebugFormat(CultureInfo.CurrentCulture, "adding topic {0} and stream to map...", topic);
// register on broker partition path changes
string partitionPath = ZooKeeperClient.DefaultBrokerTopicsPath + "/" + topic;
this.zkClient.MakeSurePersistentPathExists(partitionPath);
this.zkClient.Subscribe(partitionPath, loadBalancerListener);
}
//// register listener for session expired event
this.zkClient.Subscribe(new ZKSessionExpireListener(dirs, consumerIdString, topicCount, loadBalancerListener, this));
//// explicitly trigger load balancing for this consumer
lock (this.syncLock)
{
loadBalancerListener.SyncedRebalance();
}
return result;
}
private IDictionary <string, IList <KafkaMessageStream <TData> > > Consume <TData>(IDictionary <string, int> topicCountDict, IDecoder <TData> decoder)
{
Logger.Debug("entering consume");
if (topicCountDict == null)
{
throw new ArgumentNullException(nameof(topicCountDict));
}
var dirs = new ZKGroupDirs(this.config.GroupId);
var result = new Dictionary <string, IList <KafkaMessageStream <TData> > >();
string consumerIdString = GetConsumerIdString();
var topicCount = new TopicCount(consumerIdString, topicCountDict);
//// create a queue per topic per consumer thread
var consumerThreadIdsPerTopicMap = topicCount.GetConsumerThreadIdsPerTopic();
foreach (var topic in consumerThreadIdsPerTopicMap.Keys)
{
var streamList = new List <KafkaMessageStream <TData> >();
foreach (string threadId in consumerThreadIdsPerTopicMap[topic])
{
var stream = new BlockingCollection <FetchedDataChunk>(new ConcurrentQueue <FetchedDataChunk>());
this.queues.Add(new Tuple <string, string>(topic, threadId), stream);
streamList.Add(new KafkaMessageStream <TData>(topic, stream, this.config.Timeout, decoder));
}
result.Add(topic, streamList);
Logger.InfoFormat("adding topic {0} and stream to map...", topic);
}
// listener to consumer and partition changes
var loadBalancerListener = new ZKRebalancerListener <TData>(
this.config,
consumerIdString,
this.topicRegistry,
this.GetZkClient(),
this,
queues,
this.fetcher,
result,
topicCount);
if (this.consumerRebalanceHandler != null)
{
loadBalancerListener.ConsumerRebalance += this.consumerRebalanceHandler;
}
stopAsyncRebalancing.Add(loadBalancerListener.StopRebalance);
this.RegisterConsumerInZk(dirs, consumerIdString, topicCount);
//// register listener for session expired event
var zkSessionExpireListener = new ZKSessionExpireListener <TData>(dirs, consumerIdString, topicCount, loadBalancerListener, this);
if (this.zkSessionDisconnectedHandler != null)
{
zkSessionExpireListener.ZKSessionDisconnected += this.zkSessionDisconnectedHandler;
}
if (this.zkSessionExpiredHandler != null)
{
zkSessionExpireListener.ZKSessionExpired += this.zkSessionExpiredHandler;
}
this.GetZkClient().Subscribe(zkSessionExpireListener);
this.subscribedZookeeperStateCollection.Add(zkSessionExpireListener);
this.GetZkClient().Subscribe(dirs.ConsumerRegistryDir, loadBalancerListener);
this.subscribedChildCollection.Add(new Tuple <string, IZooKeeperChildListener>(dirs.ConsumerRegistryDir, loadBalancerListener));
result.ForEach(topicAndStreams =>
{
// register on broker partition path changes
string partitionPath = ZooKeeperClient.DefaultBrokerTopicsPath + "/" + topicAndStreams.Key;
if (this.GetZkClient().Exists(partitionPath))
{
this.GetZkClient().Subscribe(partitionPath, loadBalancerListener);
this.subscribedChildCollection.Add(new Tuple <string, IZooKeeperChildListener>(partitionPath, loadBalancerListener));
// Create a mapping of all topic partitions and their current leaders
var topicsAndPartitions = ZkUtils.GetPartitionsForTopics(this.GetZkClient(), new[] { topicAndStreams.Key });
Dictionary <string, int> partitionLeaderMap = new Dictionary <string, int>();
foreach (var partitionId in topicsAndPartitions[topicAndStreams.Key])
{
// Find/parse current partition leader for this partition and add it
// to the mapping object
var partitionStatePath = partitionPath + "/partitions/" + partitionId + "/state";
this.GetZkClient().MakeSurePersistentPathExists(partitionStatePath);
int?partitionLeader = ZkUtils.GetLeaderForPartition(this.GetZkClient(), topicAndStreams.Key, int.Parse(partitionId));
partitionLeaderMap.Add(partitionStatePath, partitionLeader.GetValueOrDefault(-1));
}
// listen for changes on the state nodes for the partitions
// this will indicate when a leader switches, or the in sync replicas change
var leaderListener = new ZkPartitionLeaderListener <TData>(loadBalancerListener, partitionLeaderMap);
foreach (var partitionId in topicsAndPartitions[topicAndStreams.Key])
{
var partitionStatePath = partitionPath + "/partitions/" + partitionId + "/state";
this.GetZkClient().Subscribe(partitionStatePath, leaderListener);
this.subscribedZookeeperDataCollection.Add(new Tuple <string, IZooKeeperDataListener>(partitionStatePath, leaderListener));
}
}
else
{
Logger.WarnFormat("The topic path at {0}, does not exist.", partitionPath);
}
});
//// explicitly trigger load balancing for this consumer
Logger.Info("Performing rebalancing. A new consumer has been added to consumer group: " + dirs.ConsumerRegistryDir + ", consumer: " + consumerIdString);
Logger.InfoFormat("Subscribe count: subscribedChildCollection:{0} , subscribedZookeeperStateCollection:{1} subscribedZookeeperDataCollection:{2} "
, subscribedChildCollection.Count, subscribedZookeeperStateCollection.Count, subscribedZookeeperDataCollection.Count);
//// When a new consumer join, need wait for rebalance finish to make sure Fetcher thread started.
loadBalancerListener.AsyncRebalance(DefaultWaitTimeForInitialRebalanceInSeconds * 1000);
return(result);
}
