internal AbstractFetcherManager(string name, string metricPrefix, int numFetchers = 1)
{
this.Name = name;
this.MetricPrefix = metricPrefix;
this.NumFetchers = numFetchers;
MetersFactory.NewGauge(
metricPrefix + "-MaxLag",
() => this.fetcherThreadMap.Aggregate(
0,
(curMaxAll, fetcherThreadMapEntry) =>
fetcherThreadMapEntry.Value.FetcherLagStats.Stats.Aggregate(
0,
(curMaxThread, fetcherLagStatsEntry) =>
(int)Math.Max(curMaxThread, fetcherLagStatsEntry.Value.Lag))));
MetersFactory.NewGauge(
metricPrefix + "-MinFetchRate",
() =>
{
var headRate = fetcherThreadMap.Any()
? fetcherThreadMap.First().Value.FetcherStats.RequestRate.OneMinuteRate()
: 0;
return(fetcherThreadMap.Aggregate(
headRate,
(curMinAll, fetcherThreadMapEntry) =>
Math.Min(curMinAll, fetcherThreadMapEntry.Value.FetcherStats.RequestRate.OneMinuteRate())));
});
}
public ProducerTopicMetrics(ClientIdAndTopic metricId)
{
this.MessageRate = MetersFactory.NewMeter(metricId + "MessagesPerSec", "messages", TimeSpan.FromSeconds(1));
this.ByteRate = MetersFactory.NewMeter(metricId + "BytesPerSec", "bytes", TimeSpan.FromSeconds(1));
this.DroppedMessageRate = MetersFactory.NewMeter(
metricId + "DroppedMessagesPerSec", "drops", TimeSpan.FromSeconds(1));
}
public ProducerStats(string clientId)
{
this.ClientId = clientId;
this.SerializationErrorRate = MetersFactory.NewMeter(
clientId + "-SerializationErrorsPerSec", "errors", TimeSpan.FromSeconds(1));
this.ResendRate = MetersFactory.NewMeter(clientId + "-ResendsPerSec", "resends", TimeSpan.FromSeconds(1));
this.FailedSendRate = MetersFactory.NewMeter(
clientId + "-FailedSendsPerSec", "failed sends", TimeSpan.FromSeconds(1));
}
public ProducerSendThread(string threadName, BlockingCollection <KeyedMessage <TKey, TValue> > queue, IEventHandler <TKey, TValue> handler, int queueTime, int batchSize, string clientId)
{
this.ThreadName = threadName;
this.Queue = queue;
this.Handler = handler;
this.QueueTime = queueTime;
this.BatchSize = batchSize;
this.ClientId = clientId;
this.backgroundWorker = new BackgroundWorker();
this.backgroundWorker.DoWork += this.Run;
MetersFactory.NewGauge(clientId + "-ProducerQueueSize", () => this.Queue.Count);
}
public FetcherStats(ClientIdAndBroker metricId)
{
this.RequestRate = MetersFactory.NewMeter(metricId + "-RequestsPerSec", "requests", TimeSpan.FromSeconds(1));
this.ByteRate = MetersFactory.NewMeter(metricId + "-BytesPerSec", "bytes", TimeSpan.FromSeconds(1));
}
public FetcherLagMetrics(ClientIdBrokerTopicPartition metricId)
{
MetersFactory.NewGauge(metricId + "-ConsumerLag", () => this.lagVal.Get());
}
public ProducerRequestMetrics(ClientIdAndBroker metricId)
{
this.RequestTimer = new KafkaTimer(MetersFactory.NewTimer(metricId + "ProducerRequestRateAndTimeMs", TimeSpan.FromMilliseconds(1), TimeSpan.FromSeconds(1)));
this.RequestSizeHist = MetersFactory.NewHistogram(metricId + "ProducerRequestSize");
}
private void ReinitializeConsumer <TKey, TValue>(
TopicCount topicCount, IList <Tuple <BlockingCollection <FetchedDataChunk>, KafkaStream <TKey, TValue> > > queuesAndStreams)
{
var dirs = new ZKGroupDirs(this.Config.GroupId);
// listener to consumer and partition changes
if (loadBalancerListener == null)
{
var topicStreamsMaps = new Dictionary <string, IList <KafkaStream <TKey, TValue> > >();
loadBalancerListener = new ZKRebalancerListener <TKey, TValue>(this, this.Config.GroupId, consumerIdString, topicStreamsMaps);
}
// create listener for session expired event if not exist yet
if (sessionExpirationListener == null)
{
sessionExpirationListener = new ZKSessionExpireListener(this,
dirs, consumerIdString, topicCount, loadBalancerListener);
}
// create listener for topic partition change event if not exist yet
if (topicPartitionChangeListener == null)
{
topicPartitionChangeListener = new ZKTopicPartitionChangeListener(this, loadBalancerListener);
}
var topicStreamsMap = (IDictionary <string, IList <KafkaStream <TKey, TValue> > >)loadBalancerListener.KafkaMessageAndMetadataStreams;
// map of {topic -> Set(thread-1, thread-2, ...)}
var consumerThreadIdsPerTopic = topicCount.GetConsumerThreadIdsPerTopic();
IList <Tuple <BlockingCollection <FetchedDataChunk>, KafkaStream <TKey, TValue> > > allQueuesAndStreams = null;
if (topicCount is WildcardTopicCount)
{
/*
* Wild-card consumption streams share the same queues, so we need to
* duplicate the list for the subsequent zip operation.
*/
allQueuesAndStreams = Enumerable.Range(1, consumerThreadIdsPerTopic.Keys.Count).SelectMany(_ => queuesAndStreams).ToList();
}
else if (topicCount is StaticTopicCount)
{
allQueuesAndStreams = queuesAndStreams;
}
var topicThreadIds = consumerThreadIdsPerTopic.SelectMany(topicAndThreadIds =>
{
var topic = topicAndThreadIds.Key;
var threadIds = topicAndThreadIds.Value;
return(threadIds.Select(id => Tuple.Create(topic, id)));
}).ToList();
Contract.Assert(topicThreadIds.Count == allQueuesAndStreams.Count, string.Format("Mismatch betwen thread ID count ({0}) adn queue count ({1})", topicThreadIds.Count, allQueuesAndStreams.Count));
var threadQueueStreamPairs = topicThreadIds.Zip(allQueuesAndStreams, Tuple.Create).ToList();
foreach (var e in threadQueueStreamPairs)
{
var topicThreadId = e.Item1;
var q = e.Item2.Item1;
topicThreadIdAndQueues[topicThreadId] = q;
Logger.DebugFormat("Adding topicThreadId {0} and queue {1} to topicThreadIdAndQueues Data structure", topicThreadId, string.Join(",", q));
MetersFactory.NewGauge(this.Config.ClientId + "-" + this.Config.GroupId + "-" + topicThreadId.Item1 + "-" + topicThreadId.Item2 + "-FetchQueueSize", () => q.Count);
}
var groupedByTopic = threadQueueStreamPairs.GroupBy(x => x.Item1.Item1).ToList();
foreach (var e in groupedByTopic)
{
var topic = e.Key;
var streams = e.Select(x => x.Item2.Item2).ToList();
topicStreamsMap[topic] = streams;
Logger.DebugFormat("adding topic {0} and {1} stream to map", topic, streams.Count);
}
// listener to consumer and partition changes
zkClient.SubscribeStateChanges(sessionExpirationListener);
zkClient.SubscribeChildChanges(dirs.ConsumerRegistryDir, loadBalancerListener);
foreach (var topicAndSteams in topicStreamsMap)
{
// register on broker partition path changes
var topicPath = ZkUtils.BrokerTopicsPath + "/" + topicAndSteams.Key;
zkClient.SubscribeDataChanges(topicPath, topicPartitionChangeListener);
}
// explicitly trigger load balancing for this consumer
loadBalancerListener.SyncedRebalance();
}