/// <summary> /// Opens connections to brokers. /// </summary> /// <param name="topicInfos"> /// The topic infos. /// </param> /// <param name="cluster"> /// The cluster. /// </param> /// <param name="queuesToBeCleared"> /// The queues to be cleared. /// </param> public void InitConnections(IEnumerable<PartitionTopicInfo> topicInfos, Cluster cluster, IEnumerable<BlockingCollection<FetchedDataChunk>> queuesToBeCleared) { this.EnsuresNotDisposed(); this.Shutdown(); if (topicInfos == null) { return; } foreach (var queueToBeCleared in queuesToBeCleared) { while (queueToBeCleared.Count > 0) { queueToBeCleared.Take(); } } var partitionTopicInfoMap = new Dictionary<int, List<PartitionTopicInfo>>(); //// re-arrange by broker id foreach (var topicInfo in topicInfos) { if (!partitionTopicInfoMap.ContainsKey(topicInfo.BrokerId)) { partitionTopicInfoMap.Add(topicInfo.BrokerId, new List<PartitionTopicInfo>() { topicInfo }); } else { partitionTopicInfoMap[topicInfo.BrokerId].Add(topicInfo); } } //// open a new fetcher thread for each broker fetcherWorkerObjects = new FetcherRunnable[partitionTopicInfoMap.Count]; int i = 0; foreach (KeyValuePair<int, List<PartitionTopicInfo>> item in partitionTopicInfoMap) { Broker broker = cluster.GetBroker(item.Key); var fetcherRunnable = new FetcherRunnable("FetcherRunnable-" + i, zkClient, config, broker, item.Value); var threadStart = new ThreadStart(fetcherRunnable.Run); var fetcherThread = new Thread(threadStart); fetcherWorkerObjects[i] = fetcherRunnable; fetcherThread.Start(); i++; } }
private bool Rebalance() { var myTopicThresdIdsMap = this.GetTopicCount(this.consumerIdString).GetConsumerThreadIdsPerTopic(); var cluster = new Cluster(zkClient); var consumersPerTopicMap = this.GetConsumersPerTopic(this.config.GroupId); var partitionsPerTopicMap = ZkUtils.GetPartitionsForTopics(this.zkClient, myTopicThresdIdsMap.Keys); var relevantTopicThreadIdsMap = GetRelevantTopicMap( myTopicThresdIdsMap, partitionsPerTopicMap, this.oldPartitionsPerTopicMap, consumersPerTopicMap, this.oldConsumersPerTopicMap); if (relevantTopicThreadIdsMap.Count <= 0) { Logger.InfoFormat(CultureInfo.CurrentCulture, "Consumer {0} with {1} doesn't need to rebalance.", this.consumerIdString, consumersPerTopicMap); return true; } Logger.Info("Committing all offsets"); this.zkConsumerConnector.CommitOffsets(); Logger.Info("Releasing parittion ownership"); this.ReleasePartitionOwnership(); var queuesToBeCleared = new List<BlockingCollection<FetchedDataChunk>>(); foreach (var item in relevantTopicThreadIdsMap) { this.topicRegistry.Remove(item.Key); this.topicRegistry.Add(item.Key, new Dictionary<Partition, PartitionTopicInfo>()); var topicDirs = new ZKGroupTopicDirs(config.GroupId, item.Key); var curConsumers = consumersPerTopicMap[item.Key]; var curPartitions = new List<string>(partitionsPerTopicMap[item.Key]); var numberOfPartsPerConsumer = curPartitions.Count / curConsumers.Count; var numberOfConsumersWithExtraPart = curPartitions.Count % curConsumers.Count; Logger.InfoFormat( CultureInfo.CurrentCulture, "Consumer {0} rebalancing the following partitions: {1} for topic {2} with consumers: {3}", this.consumerIdString, string.Join(",", curPartitions), item.Key, string.Join(",", curConsumers)); foreach (string consumerThreadId in item.Value) { var myConsumerPosition = curConsumers.IndexOf(consumerThreadId); if (myConsumerPosition < 0) { continue; } var startPart = (numberOfPartsPerConsumer * myConsumerPosition) + Math.Min(myConsumerPosition, numberOfConsumersWithExtraPart); var numberOfParts = numberOfPartsPerConsumer + (myConsumerPosition + 1 > numberOfConsumersWithExtraPart ? 0 : 1); if (numberOfParts <= 0) { Logger.WarnFormat(CultureInfo.CurrentCulture, "No broker partitions consumed by consumer thread {0} for topic {1}", consumerThreadId, item.Key); } else { for (int i = startPart; i < startPart + numberOfParts; i++) { var partition = curPartitions[i]; Logger.InfoFormat(CultureInfo.CurrentCulture, "{0} attempting to claim partition {1}", consumerThreadId, partition); if (!this.ProcessPartition(topicDirs, partition, item.Key, consumerThreadId)) { return false; } } queuesToBeCleared.Add(queues[new Tuple<string, string>(item.Key, consumerThreadId)]); } } } this.UpdateFetcher(cluster, queuesToBeCleared); this.oldPartitionsPerTopicMap = partitionsPerTopicMap; this.oldConsumersPerTopicMap = consumersPerTopicMap; return true; }
private void UpdateFetcher(Cluster cluster, IEnumerable<BlockingCollection<FetchedDataChunk>> queuesToBeCleared) { var allPartitionInfos = new List<PartitionTopicInfo>(); foreach (var item in this.topicRegistry.Values) { foreach (var partitionTopicInfo in item.Values) { allPartitionInfos.Add(partitionTopicInfo); } } Logger.InfoFormat( CultureInfo.CurrentCulture, "Consumer {0} selected partitions: {1}", this.consumerIdString, string.Join(",", allPartitionInfos.OrderBy(x => x.Partition.Name).Select(y => y.Partition.Name))); if (this.fetcher != null) { this.fetcher.InitConnections(allPartitionInfos, cluster, queuesToBeCleared); } }
private void RefreshKafkaBrokersInfo() { this.kafkaCluster = new Cluster(this.zkClient); }