/// <summary>
/// Get consumer instance for specified broker from cache. Or create a consumer if it does not already exist.
/// </summary>
/// <param name="brokerId">Broker id</param>
/// <returns>Consumer instance for specified broker.</returns>
private Consumer GetConsumer(int brokerId)
{
Consumer result;
if (!consumerDict.TryGetValue(brokerId, out result))
{
var broker = kafkaCluster.GetBroker(brokerId);
if (broker == null)
{
throw new BrokerNotAvailableException(brokerId);
}
result = new Consumer(config, broker.Host, broker.Port);
consumerDict.Add(brokerId, result);
}
return(result);
}
/// <summary>
/// Opens connections to brokers.
/// </summary>
/// <param name="topicInfos">
/// The topic infos.
/// </param>
/// <param name="cluster">
/// The cluster.
/// </param>
/// <param name="queuesToBeCleared">
/// The queues to be cleared.
/// </param>
public void InitConnections(IEnumerable <PartitionTopicInfo> topicInfos, Cluster.Cluster cluster)
{
EnsuresNotDisposed();
Shutdown();
if (topicInfos == null)
{
return;
}
var partitionTopicInfoMap = new Dictionary <int, List <PartitionTopicInfo> >();
//// re-arrange by broker id
foreach (var topicInfo in topicInfos)
{
if (!partitionTopicInfoMap.ContainsKey(topicInfo.BrokerId))
{
partitionTopicInfoMap.Add(topicInfo.BrokerId, new List <PartitionTopicInfo> {
topicInfo
});
}
else
{
partitionTopicInfoMap[topicInfo.BrokerId].Add(topicInfo);
}
}
_leaderFinder = new PartitionLeaderFinder(_partitionsNeedingLeaders, cluster, _config, CreateFetchThread);
_leaderFinderThread = new Thread(_leaderFinder.Start);
_leaderFinderThread.Start();
//// open a new fetcher thread for each broker
_fetcherWorkerObjects = new ConcurrentBag <FetcherRunnable>();
_fetcherThreads = new ConcurrentBag <Thread>();
var i = 0;
foreach (var item in partitionTopicInfoMap)
{
var broker = cluster.GetBroker(item.Key);
if (broker == null)
{
foreach (var p in item.Value)
{
AddPartitionWithError(p);
}
Logger.Error("Could not find broker associated with broker id: " + item.Key + " partitions: " +
string.Join(",",
item.Value.Select(r => string.Format("Topic:{0} PartitionsID:{1} ", r.Topic,
r.PartitionId))
.ToArray()) + " will repeat retry ...");
}
else
{
Logger.Debug("Found broker associated with broker id: " + item.Key + " partitions: " +
string.Join(",",
item.Value.Select(r => string.Format("Topic:{0} PartitionsID:{1} ", r.Topic,
r.PartitionId))
.ToArray()) + " will create fetch threads ...");
CreateFetchThread(item.Value, broker);
}
i++;
}
}