/// <summary>
/// Opens connections to brokers.
/// </summary>
/// <param name="topicInfos">
/// The topic infos.
/// </param>
/// <param name="cluster">
/// The cluster.
/// </param>
/// <param name="queuesToBeCleared">
/// The queues to be cleared.
/// </param>
public void InitConnections(IEnumerable <PartitionTopicInfo> topicInfos, Cluster cluster, IEnumerable <BlockingCollection <FetchedDataChunk> > queuesToBeCleared)
{
this.EnsuresNotDisposed();
this.Shutdown();
if (topicInfos == null)
{
return;
}
foreach (var queueToBeCleared in queuesToBeCleared)
{
while (queueToBeCleared.Count > 0)
{
queueToBeCleared.Take();
}
}
var partitionTopicInfoMap = new Dictionary <int, List <PartitionTopicInfo> >();
//// re-arrange by broker id
foreach (var topicInfo in topicInfos)
{
if (!partitionTopicInfoMap.ContainsKey(topicInfo.BrokerId))
{
partitionTopicInfoMap.Add(topicInfo.BrokerId, new List <PartitionTopicInfo>()
{
topicInfo
});
}
else
{
partitionTopicInfoMap[topicInfo.BrokerId].Add(topicInfo);
}
}
//// open a new fetcher thread for each broker
fetcherWorkerObjects = new FetcherRunnable[partitionTopicInfoMap.Count];
int i = 0;
foreach (KeyValuePair <int, List <PartitionTopicInfo> > item in partitionTopicInfoMap)
{
Broker broker = cluster.GetBroker(item.Key);
var fetcherRunnable = new FetcherRunnable("FetcherRunnable-" + i, zkClient, config, broker, item.Value);
var threadStart = new ThreadStart(fetcherRunnable.Run);
var fetcherThread = new Thread(threadStart);
fetcherWorkerObjects[i] = fetcherRunnable;
fetcherThread.Start();
i++;
}
}
private void CreateFetchThread(List <PartitionTopicInfo> partitions, Broker broker)
{
if (_disposed)
{
return;
}
Logger.DebugFormat("Creating Fetcher on broker {0} for partitions: {1}", broker.Id, string.Join(",", partitions.Select(p => string.Format("{0}({1})", p.Topic, p.PartitionId))));
var fetcherRunnable = new FetcherRunnable("FetcherRunnable-" + _fetcherWorkerObjects.Count, _zkClient, _config, broker, partitions, AddPartitionWithError);
var threadStart = new ThreadStart(fetcherRunnable.Run);
var fetcherThread = new Thread(threadStart);
_fetcherWorkerObjects.Add(fetcherRunnable);
fetcherThread.Name = string.Format("FetcherThread_broker_{0}_partitions_{1}", broker.Id, string.Join(",", partitions.Select(p => string.Format("{0}({1})", p.Topic, p.PartitionId))));
_fetcherThreads.Add(fetcherThread);
fetcherThread.Start();
}
/// <summary>
/// Opens connections to brokers.
/// </summary>
/// <param name="topicInfos">
/// The topic infos.
/// </param>
/// <param name="cluster">
/// The cluster.
/// </param>
/// <param name="queuesToBeCleared">
/// The queues to be cleared.
/// </param>
public void InitConnections(IEnumerable<PartitionTopicInfo> topicInfos, Cluster cluster, IEnumerable<BlockingCollection<FetchedDataChunk>> queuesToBeCleared)
{
this.EnsuresNotDisposed();
this.Shutdown();
if (topicInfos == null)
{
return;
}
foreach (var queueToBeCleared in queuesToBeCleared)
{
while (queueToBeCleared.Count > 0)
{
queueToBeCleared.Take();
}
}
var partitionTopicInfoMap = new Dictionary<int, List<PartitionTopicInfo>>();
//// re-arrange by broker id
foreach (var topicInfo in topicInfos)
{
if (!partitionTopicInfoMap.ContainsKey(topicInfo.BrokerId))
{
partitionTopicInfoMap.Add(topicInfo.BrokerId, new List<PartitionTopicInfo>() { topicInfo });
}
else
{
partitionTopicInfoMap[topicInfo.BrokerId].Add(topicInfo);
}
}
//// open a new fetcher thread for each broker
fetcherWorkerObjects = new FetcherRunnable[partitionTopicInfoMap.Count];
int i = 0;
foreach (KeyValuePair<int, List<PartitionTopicInfo>> item in partitionTopicInfoMap)
{
Broker broker = cluster.GetBroker(item.Key);
var fetcherRunnable = new FetcherRunnable("FetcherRunnable-" + i, zkClient, config, broker, item.Value);
var threadStart = new ThreadStart(fetcherRunnable.Run);
var fetcherThread = new Thread(threadStart);
fetcherWorkerObjects[i] = fetcherRunnable;
fetcherThread.Start();
i++;
}
}
