/// <summary> /// Opens connections to brokers. /// </summary> /// <param name="topicInfos"> /// The topic infos. /// </param> /// <param name="cluster"> /// The cluster. /// </param> /// <param name="queuesToBeCleared"> /// The queues to be cleared. /// </param> public void InitConnections(IEnumerable <PartitionTopicInfo> topicInfos, Cluster cluster, IEnumerable <BlockingCollection <FetchedDataChunk> > queuesToBeCleared) { this.EnsuresNotDisposed(); this.Shutdown(); if (topicInfos == null) { return; } foreach (var queueToBeCleared in queuesToBeCleared) { while (queueToBeCleared.Count > 0) { queueToBeCleared.Take(); } } var partitionTopicInfoMap = new Dictionary <int, List <PartitionTopicInfo> >(); //// re-arrange by broker id foreach (var topicInfo in topicInfos) { if (!partitionTopicInfoMap.ContainsKey(topicInfo.BrokerId)) { partitionTopicInfoMap.Add(topicInfo.BrokerId, new List <PartitionTopicInfo>() { topicInfo }); } else { partitionTopicInfoMap[topicInfo.BrokerId].Add(topicInfo); } } //// open a new fetcher thread for each broker fetcherWorkerObjects = new FetcherRunnable[partitionTopicInfoMap.Count]; int i = 0; foreach (KeyValuePair <int, List <PartitionTopicInfo> > item in partitionTopicInfoMap) { Broker broker = cluster.GetBroker(item.Key); var fetcherRunnable = new FetcherRunnable("FetcherRunnable-" + i, zkClient, config, broker, item.Value); var threadStart = new ThreadStart(fetcherRunnable.Run); var fetcherThread = new Thread(threadStart); fetcherWorkerObjects[i] = fetcherRunnable; fetcherThread.Start(); i++; } }
private void CreateFetchThread(List <PartitionTopicInfo> partitions, Broker broker) { if (_disposed) { return; } Logger.DebugFormat("Creating Fetcher on broker {0} for partitions: {1}", broker.Id, string.Join(",", partitions.Select(p => string.Format("{0}({1})", p.Topic, p.PartitionId)))); var fetcherRunnable = new FetcherRunnable("FetcherRunnable-" + _fetcherWorkerObjects.Count, _zkClient, _config, broker, partitions, AddPartitionWithError); var threadStart = new ThreadStart(fetcherRunnable.Run); var fetcherThread = new Thread(threadStart); _fetcherWorkerObjects.Add(fetcherRunnable); fetcherThread.Name = string.Format("FetcherThread_broker_{0}_partitions_{1}", broker.Id, string.Join(",", partitions.Select(p => string.Format("{0}({1})", p.Topic, p.PartitionId)))); _fetcherThreads.Add(fetcherThread); fetcherThread.Start(); }
/// <summary> /// Opens connections to brokers. /// </summary> /// <param name="topicInfos"> /// The topic infos. /// </param> /// <param name="cluster"> /// The cluster. /// </param> /// <param name="queuesToBeCleared"> /// The queues to be cleared. /// </param> public void InitConnections(IEnumerable<PartitionTopicInfo> topicInfos, Cluster cluster, IEnumerable<BlockingCollection<FetchedDataChunk>> queuesToBeCleared) { this.EnsuresNotDisposed(); this.Shutdown(); if (topicInfos == null) { return; } foreach (var queueToBeCleared in queuesToBeCleared) { while (queueToBeCleared.Count > 0) { queueToBeCleared.Take(); } } var partitionTopicInfoMap = new Dictionary<int, List<PartitionTopicInfo>>(); //// re-arrange by broker id foreach (var topicInfo in topicInfos) { if (!partitionTopicInfoMap.ContainsKey(topicInfo.BrokerId)) { partitionTopicInfoMap.Add(topicInfo.BrokerId, new List<PartitionTopicInfo>() { topicInfo }); } else { partitionTopicInfoMap[topicInfo.BrokerId].Add(topicInfo); } } //// open a new fetcher thread for each broker fetcherWorkerObjects = new FetcherRunnable[partitionTopicInfoMap.Count]; int i = 0; foreach (KeyValuePair<int, List<PartitionTopicInfo>> item in partitionTopicInfoMap) { Broker broker = cluster.GetBroker(item.Key); var fetcherRunnable = new FetcherRunnable("FetcherRunnable-" + i, zkClient, config, broker, item.Value); var threadStart = new ThreadStart(fetcherRunnable.Run); var fetcherThread = new Thread(threadStart); fetcherWorkerObjects[i] = fetcherRunnable; fetcherThread.Start(); i++; } }