/// <summary>
/// Opens connections to brokers.
/// </summary>
/// <param name="topicInfos">
/// The topic infos.
/// </param>
/// <param name="cluster">
/// The cluster.
/// </param>
/// <param name="queuesToBeCleared">
/// The queues to be cleared.
/// </param>
public void InitConnections(IEnumerable<PartitionTopicInfo> topicInfos, Cluster cluster, IEnumerable<BlockingCollection<FetchedDataChunk>> queuesToBeCleared)
{
this.EnsuresNotDisposed();
this.Shutdown();
if (topicInfos == null)
{
return;
}
foreach (var queueToBeCleared in queuesToBeCleared)
{
while (queueToBeCleared.Count > 0)
{
queueToBeCleared.Take();
}
}
var partitionTopicInfoMap = new Dictionary<int, List<PartitionTopicInfo>>();
//// re-arrange by broker id
foreach (var topicInfo in topicInfos)
{
if (!partitionTopicInfoMap.ContainsKey(topicInfo.BrokerId))
{
partitionTopicInfoMap.Add(topicInfo.BrokerId, new List<PartitionTopicInfo>() { topicInfo });
}
else
{
partitionTopicInfoMap[topicInfo.BrokerId].Add(topicInfo);
}
}
//// open a new fetcher thread for each broker
fetcherWorkerObjects = new FetcherRunnable[partitionTopicInfoMap.Count];
int i = 0;
foreach (KeyValuePair<int, List<PartitionTopicInfo>> item in partitionTopicInfoMap)
{
Broker broker = cluster.GetBroker(item.Key);
var fetcherRunnable = new FetcherRunnable("FetcherRunnable-" + i, zkClient, config, broker, item.Value);
var threadStart = new ThreadStart(fetcherRunnable.Run);
var fetcherThread = new Thread(threadStart);
fetcherWorkerObjects[i] = fetcherRunnable;
fetcherThread.Start();
i++;
}
}
private bool Rebalance()
{
var myTopicThresdIdsMap = this.GetTopicCount(this.consumerIdString).GetConsumerThreadIdsPerTopic();
var cluster = new Cluster(zkClient);
var consumersPerTopicMap = this.GetConsumersPerTopic(this.config.GroupId);
var partitionsPerTopicMap = ZkUtils.GetPartitionsForTopics(this.zkClient, myTopicThresdIdsMap.Keys);
var relevantTopicThreadIdsMap = GetRelevantTopicMap(
myTopicThresdIdsMap,
partitionsPerTopicMap,
this.oldPartitionsPerTopicMap,
consumersPerTopicMap,
this.oldConsumersPerTopicMap);
if (relevantTopicThreadIdsMap.Count <= 0)
{
Logger.InfoFormat(CultureInfo.CurrentCulture, "Consumer {0} with {1} doesn't need to rebalance.", this.consumerIdString, consumersPerTopicMap);
return true;
}
Logger.Info("Committing all offsets");
this.zkConsumerConnector.CommitOffsets();
Logger.Info("Releasing parittion ownership");
this.ReleasePartitionOwnership();
var queuesToBeCleared = new List<BlockingCollection<FetchedDataChunk>>();
foreach (var item in relevantTopicThreadIdsMap)
{
this.topicRegistry.Remove(item.Key);
this.topicRegistry.Add(item.Key, new Dictionary<Partition, PartitionTopicInfo>());
var topicDirs = new ZKGroupTopicDirs(config.GroupId, item.Key);
var curConsumers = consumersPerTopicMap[item.Key];
var curPartitions = new List<string>(partitionsPerTopicMap[item.Key]);
var numberOfPartsPerConsumer = curPartitions.Count / curConsumers.Count;
var numberOfConsumersWithExtraPart = curPartitions.Count % curConsumers.Count;
Logger.InfoFormat(
CultureInfo.CurrentCulture,
"Consumer {0} rebalancing the following partitions: {1} for topic {2} with consumers: {3}",
this.consumerIdString,
string.Join(",", curPartitions),
item.Key,
string.Join(",", curConsumers));
foreach (string consumerThreadId in item.Value)
{
var myConsumerPosition = curConsumers.IndexOf(consumerThreadId);
if (myConsumerPosition < 0)
{
continue;
}
var startPart = (numberOfPartsPerConsumer * myConsumerPosition) +
Math.Min(myConsumerPosition, numberOfConsumersWithExtraPart);
var numberOfParts = numberOfPartsPerConsumer + (myConsumerPosition + 1 > numberOfConsumersWithExtraPart ? 0 : 1);
if (numberOfParts <= 0)
{
Logger.WarnFormat(CultureInfo.CurrentCulture, "No broker partitions consumed by consumer thread {0} for topic {1}", consumerThreadId, item.Key);
}
else
{
for (int i = startPart; i < startPart + numberOfParts; i++)
{
var partition = curPartitions[i];
Logger.InfoFormat(CultureInfo.CurrentCulture, "{0} attempting to claim partition {1}", consumerThreadId, partition);
if (!this.ProcessPartition(topicDirs, partition, item.Key, consumerThreadId))
{
return false;
}
}
queuesToBeCleared.Add(queues[new Tuple<string, string>(item.Key, consumerThreadId)]);
}
}
}
this.UpdateFetcher(cluster, queuesToBeCleared);
this.oldPartitionsPerTopicMap = partitionsPerTopicMap;
this.oldConsumersPerTopicMap = consumersPerTopicMap;
return true;
}
private void UpdateFetcher(Cluster cluster, IEnumerable<BlockingCollection<FetchedDataChunk>> queuesToBeCleared)
{
var allPartitionInfos = new List<PartitionTopicInfo>();
foreach (var item in this.topicRegistry.Values)
{
foreach (var partitionTopicInfo in item.Values)
{
allPartitionInfos.Add(partitionTopicInfo);
}
}
Logger.InfoFormat(
CultureInfo.CurrentCulture,
"Consumer {0} selected partitions: {1}",
this.consumerIdString,
string.Join(",", allPartitionInfos.OrderBy(x => x.Partition.Name).Select(y => y.Partition.Name)));
if (this.fetcher != null)
{
this.fetcher.InitConnections(allPartitionInfos, cluster, queuesToBeCleared);
}
}
private void RefreshKafkaBrokersInfo()
{
this.kafkaCluster = new Cluster(this.zkClient);
}
