/// <summary>
/// Opens connections to brokers.
/// </summary>
/// <param name="topicInfos">
/// The topic infos.
/// </param>
/// <param name="cluster">
/// The cluster.
/// </param>
/// <param name="queuesToBeCleared">
/// The queues to be cleared.
/// </param>
public void InitConnections(IEnumerable <PartitionTopicInfo> topicInfos, Cluster cluster)
{
EnsuresNotDisposed();
Shutdown();
if (topicInfos == null)
{
return;
}
var partitionTopicInfoMap = new Dictionary <int, List <PartitionTopicInfo> >();
//// re-arrange by broker id
foreach (PartitionTopicInfo topicInfo in topicInfos)
{
if (!partitionTopicInfoMap.ContainsKey(topicInfo.BrokerId))
{
partitionTopicInfoMap.Add(topicInfo.BrokerId, new List <PartitionTopicInfo> {
topicInfo
});
}
else
{
partitionTopicInfoMap[topicInfo.BrokerId].Add(topicInfo);
}
}
_leaderFinder = new PartitionLeaderFinder(_partitionsNeedingLeaders, cluster, _config, CreateFetchThread);
_leaderFinderThread = new Thread(_leaderFinder.Start);
_leaderFinderThread.Start();
//// open a new fetcher thread for each broker
_fetcherWorkerObjects = new ConcurrentBag <FetcherRunnable>();
_fetcherThreads = new ConcurrentBag <Thread>();
int i = 0;
foreach (KeyValuePair <int, List <PartitionTopicInfo> > item in partitionTopicInfoMap)
{
Broker broker = cluster.GetBroker(item.Key);
if (broker == null)
{
foreach (PartitionTopicInfo p in item.Value)
{
AddPartitionWithError(p);
}
Logger.Error("Could not find broker associated with broker id: " + item.Key + " partitions: " + string.Join(",", item.Value.Select(r => string.Format("Topic:{0} PartitionsID:{1} ", r.Topic, r.PartitionId)).ToArray()) + " will repeat retry ...");
}
else
{
Logger.Debug("Found broker associated with broker id: " + item.Key + " partitions: " + string.Join(",", item.Value.Select(r => string.Format("Topic:{0} PartitionsID:{1} ", r.Topic, r.PartitionId)).ToArray()) + " will create fetch threads ...");
CreateFetchThread(item.Value, broker);
}
i++;
}
}
/// <summary>
/// Shuts down all fetch threads
/// </summary>
public void Shutdown()
{
if (_fetcherWorkerObjects != null)
{
foreach (var fetcherRunnable in _fetcherWorkerObjects)
{
if (fetcherRunnable == null)
{
Logger.Error("Fetch Runnable is null!");
}
else
{
fetcherRunnable.Shutdown();
}
}
var threadsStillRunning = 0;
var stopWatch = Stopwatch.StartNew();
var shutdownTimeout = false;
// make sure all fetcher threads stopped
do
{
Thread.Sleep(500);
threadsStillRunning = 0;
foreach (var fetcherThread in _fetcherThreads)
{
if (fetcherThread == null)
{
Logger.Error("Fetch thread is null!");
}
else
{
if (fetcherThread.IsAlive)
{
threadsStillRunning++;
}
}
}
if (stopWatch.ElapsedMilliseconds >= _config.ShutdownTimeout)
{
shutdownTimeout = true;
}
} while (threadsStillRunning > 0 && !shutdownTimeout);
stopWatch.Stop();
if (shutdownTimeout)
{
// BUG:1482409 - added timeout watch and forceful aborting of lingering background threads.
// shutdown exceeded timeout
Logger.Warn(
"All background fetcher threads did not shutdown in the specified amount of time. Raising abort exceptions to stop them.");
foreach (var fetcherThread in _fetcherThreads)
{
if (fetcherThread == null)
{
Logger.Error("Fetch thread is null!");
}
if (fetcherThread.IsAlive)
{
fetcherThread.Abort();
}
}
}
_leaderFinder.Stop();
_leaderFinder = null;
_leaderFinderThread = null;
_fetcherWorkerObjects = null;
_fetcherThreads = null;
}
}
