internal AbstractFetcherManager(string name, string metricPrefix, int numFetchers = 1) { this.Name = name; this.MetricPrefix = metricPrefix; this.NumFetchers = numFetchers; MetersFactory.NewGauge( metricPrefix + "-MaxLag", () => this.fetcherThreadMap.Aggregate( 0, (curMaxAll, fetcherThreadMapEntry) => fetcherThreadMapEntry.Value.FetcherLagStats.Stats.Aggregate( 0, (curMaxThread, fetcherLagStatsEntry) => (int)Math.Max(curMaxThread, fetcherLagStatsEntry.Value.Lag)))); MetersFactory.NewGauge( metricPrefix + "-MinFetchRate", () => { var headRate = fetcherThreadMap.Any() ? fetcherThreadMap.First().Value.FetcherStats.RequestRate.OneMinuteRate() : 0; return(fetcherThreadMap.Aggregate( headRate, (curMinAll, fetcherThreadMapEntry) => Math.Min(curMinAll, fetcherThreadMapEntry.Value.FetcherStats.RequestRate.OneMinuteRate()))); }); }
public ProducerTopicMetrics(ClientIdAndTopic metricId) { this.MessageRate = MetersFactory.NewMeter(metricId + "MessagesPerSec", "messages", TimeSpan.FromSeconds(1)); this.ByteRate = MetersFactory.NewMeter(metricId + "BytesPerSec", "bytes", TimeSpan.FromSeconds(1)); this.DroppedMessageRate = MetersFactory.NewMeter( metricId + "DroppedMessagesPerSec", "drops", TimeSpan.FromSeconds(1)); }
public ProducerStats(string clientId) { this.ClientId = clientId; this.SerializationErrorRate = MetersFactory.NewMeter( clientId + "-SerializationErrorsPerSec", "errors", TimeSpan.FromSeconds(1)); this.ResendRate = MetersFactory.NewMeter(clientId + "-ResendsPerSec", "resends", TimeSpan.FromSeconds(1)); this.FailedSendRate = MetersFactory.NewMeter( clientId + "-FailedSendsPerSec", "failed sends", TimeSpan.FromSeconds(1)); }
public ProducerSendThread(string threadName, BlockingCollection <KeyedMessage <TKey, TValue> > queue, IEventHandler <TKey, TValue> handler, int queueTime, int batchSize, string clientId) { this.ThreadName = threadName; this.Queue = queue; this.Handler = handler; this.QueueTime = queueTime; this.BatchSize = batchSize; this.ClientId = clientId; this.backgroundWorker = new BackgroundWorker(); this.backgroundWorker.DoWork += this.Run; MetersFactory.NewGauge(clientId + "-ProducerQueueSize", () => this.Queue.Count); }
public FetcherStats(ClientIdAndBroker metricId) { this.RequestRate = MetersFactory.NewMeter(metricId + "-RequestsPerSec", "requests", TimeSpan.FromSeconds(1)); this.ByteRate = MetersFactory.NewMeter(metricId + "-BytesPerSec", "bytes", TimeSpan.FromSeconds(1)); }
public FetcherLagMetrics(ClientIdBrokerTopicPartition metricId) { MetersFactory.NewGauge(metricId + "-ConsumerLag", () => this.lagVal.Get()); }
public ProducerRequestMetrics(ClientIdAndBroker metricId) { this.RequestTimer = new KafkaTimer(MetersFactory.NewTimer(metricId + "ProducerRequestRateAndTimeMs", TimeSpan.FromMilliseconds(1), TimeSpan.FromSeconds(1))); this.RequestSizeHist = MetersFactory.NewHistogram(metricId + "ProducerRequestSize"); }
private void ReinitializeConsumer <TKey, TValue>( TopicCount topicCount, IList <Tuple <BlockingCollection <FetchedDataChunk>, KafkaStream <TKey, TValue> > > queuesAndStreams) { var dirs = new ZKGroupDirs(this.Config.GroupId); // listener to consumer and partition changes if (loadBalancerListener == null) { var topicStreamsMaps = new Dictionary <string, IList <KafkaStream <TKey, TValue> > >(); loadBalancerListener = new ZKRebalancerListener <TKey, TValue>(this, this.Config.GroupId, consumerIdString, topicStreamsMaps); } // create listener for session expired event if not exist yet if (sessionExpirationListener == null) { sessionExpirationListener = new ZKSessionExpireListener(this, dirs, consumerIdString, topicCount, loadBalancerListener); } // create listener for topic partition change event if not exist yet if (topicPartitionChangeListener == null) { topicPartitionChangeListener = new ZKTopicPartitionChangeListener(this, loadBalancerListener); } var topicStreamsMap = (IDictionary <string, IList <KafkaStream <TKey, TValue> > >)loadBalancerListener.KafkaMessageAndMetadataStreams; // map of {topic -> Set(thread-1, thread-2, ...)} var consumerThreadIdsPerTopic = topicCount.GetConsumerThreadIdsPerTopic(); IList <Tuple <BlockingCollection <FetchedDataChunk>, KafkaStream <TKey, TValue> > > allQueuesAndStreams = null; if (topicCount is WildcardTopicCount) { /* * Wild-card consumption streams share the same queues, so we need to * duplicate the list for the subsequent zip operation. */ allQueuesAndStreams = Enumerable.Range(1, consumerThreadIdsPerTopic.Keys.Count).SelectMany(_ => queuesAndStreams).ToList(); } else if (topicCount is StaticTopicCount) { allQueuesAndStreams = queuesAndStreams; } var topicThreadIds = consumerThreadIdsPerTopic.SelectMany(topicAndThreadIds => { var topic = topicAndThreadIds.Key; var threadIds = topicAndThreadIds.Value; return(threadIds.Select(id => Tuple.Create(topic, id))); }).ToList(); Contract.Assert(topicThreadIds.Count == allQueuesAndStreams.Count, string.Format("Mismatch betwen thread ID count ({0}) adn queue count ({1})", topicThreadIds.Count, allQueuesAndStreams.Count)); var threadQueueStreamPairs = topicThreadIds.Zip(allQueuesAndStreams, Tuple.Create).ToList(); foreach (var e in threadQueueStreamPairs) { var topicThreadId = e.Item1; var q = e.Item2.Item1; topicThreadIdAndQueues[topicThreadId] = q; Logger.DebugFormat("Adding topicThreadId {0} and queue {1} to topicThreadIdAndQueues Data structure", topicThreadId, string.Join(",", q)); MetersFactory.NewGauge(this.Config.ClientId + "-" + this.Config.GroupId + "-" + topicThreadId.Item1 + "-" + topicThreadId.Item2 + "-FetchQueueSize", () => q.Count); } var groupedByTopic = threadQueueStreamPairs.GroupBy(x => x.Item1.Item1).ToList(); foreach (var e in groupedByTopic) { var topic = e.Key; var streams = e.Select(x => x.Item2.Item2).ToList(); topicStreamsMap[topic] = streams; Logger.DebugFormat("adding topic {0} and {1} stream to map", topic, streams.Count); } // listener to consumer and partition changes zkClient.SubscribeStateChanges(sessionExpirationListener); zkClient.SubscribeChildChanges(dirs.ConsumerRegistryDir, loadBalancerListener); foreach (var topicAndSteams in topicStreamsMap) { // register on broker partition path changes var topicPath = ZkUtils.BrokerTopicsPath + "/" + topicAndSteams.Key; zkClient.SubscribeDataChanges(topicPath, topicPartitionChangeListener); } // explicitly trigger load balancing for this consumer loadBalancerListener.SyncedRebalance(); }