/// <summary> /// Opens connections to brokers. /// </summary> /// <param name="topicInfos"> /// The topic infos. /// </param> /// <param name="cluster"> /// The cluster. /// </param> /// <param name="queuesToBeCleared"> /// The queues to be cleared. /// </param> public void InitConnections(IEnumerable <PartitionTopicInfo> topicInfos, Cluster cluster) { EnsuresNotDisposed(); Shutdown(); if (topicInfos == null) { return; } var partitionTopicInfoMap = new Dictionary <int, List <PartitionTopicInfo> >(); //// re-arrange by broker id foreach (PartitionTopicInfo topicInfo in topicInfos) { if (!partitionTopicInfoMap.ContainsKey(topicInfo.BrokerId)) { partitionTopicInfoMap.Add(topicInfo.BrokerId, new List <PartitionTopicInfo> { topicInfo }); } else { partitionTopicInfoMap[topicInfo.BrokerId].Add(topicInfo); } } _leaderFinder = new PartitionLeaderFinder(_partitionsNeedingLeaders, cluster, _config, CreateFetchThread); _leaderFinderThread = new Thread(_leaderFinder.Start); _leaderFinderThread.Start(); //// open a new fetcher thread for each broker _fetcherWorkerObjects = new ConcurrentBag <FetcherRunnable>(); _fetcherThreads = new ConcurrentBag <Thread>(); int i = 0; foreach (KeyValuePair <int, List <PartitionTopicInfo> > item in partitionTopicInfoMap) { Broker broker = cluster.GetBroker(item.Key); if (broker == null) { foreach (PartitionTopicInfo p in item.Value) { AddPartitionWithError(p); } Logger.Error("Could not find broker associated with broker id: " + item.Key + " partitions: " + string.Join(",", item.Value.Select(r => string.Format("Topic:{0} PartitionsID:{1} ", r.Topic, r.PartitionId)).ToArray()) + " will repeat retry ..."); } else { Logger.Debug("Found broker associated with broker id: " + item.Key + " partitions: " + string.Join(",", item.Value.Select(r => string.Format("Topic:{0} PartitionsID:{1} ", r.Topic, r.PartitionId)).ToArray()) + " will create fetch threads ..."); CreateFetchThread(item.Value, broker); } i++; } }
/// <summary> /// Shuts down all fetch threads /// </summary> public void Shutdown() { if (_fetcherWorkerObjects != null) { foreach (var fetcherRunnable in _fetcherWorkerObjects) { if (fetcherRunnable == null) { Logger.Error("Fetch Runnable is null!"); } else { fetcherRunnable.Shutdown(); } } var threadsStillRunning = 0; var stopWatch = Stopwatch.StartNew(); var shutdownTimeout = false; // make sure all fetcher threads stopped do { Thread.Sleep(500); threadsStillRunning = 0; foreach (var fetcherThread in _fetcherThreads) { if (fetcherThread == null) { Logger.Error("Fetch thread is null!"); } else { if (fetcherThread.IsAlive) { threadsStillRunning++; } } } if (stopWatch.ElapsedMilliseconds >= _config.ShutdownTimeout) { shutdownTimeout = true; } } while (threadsStillRunning > 0 && !shutdownTimeout); stopWatch.Stop(); if (shutdownTimeout) { // BUG:1482409 - added timeout watch and forceful aborting of lingering background threads. // shutdown exceeded timeout Logger.Warn( "All background fetcher threads did not shutdown in the specified amount of time. Raising abort exceptions to stop them."); foreach (var fetcherThread in _fetcherThreads) { if (fetcherThread == null) { Logger.Error("Fetch thread is null!"); } if (fetcherThread.IsAlive) { fetcherThread.Abort(); } } } _leaderFinder.Stop(); _leaderFinder = null; _leaderFinderThread = null; _fetcherWorkerObjects = null; _fetcherThreads = null; } }