private void PublishTopicsStats(Dictionary <string, TopicStatistic> statisticsTopics)
        {
            long now = DateTime.Now.GetMilliseconds();

            PublishTopicsStatistics(statisticsTopics);

            foreach (var topic in statisticsTopics)
            {
                foreach (var partition in topic.Value.Partitions)
                {
                    LibrdKafkaSensor.ScopedLibrdKafkaSensor.Record(PartitionTotalNumberOfMessagesProducedSensor
                                                                   .Scoped(
                                                                       (LibrdKafkaBaseMetrics.TOPIC_TAG, topic.Value.TopicName),
                                                                       (LibrdKafkaBaseMetrics.BROKER_ID_TAG, partition.Value.BrokerId.ToString()),
                                                                       (LibrdKafkaBaseMetrics.PARTITION_ID_TAG, partition.Value.PartitionId.ToString()))
                                                                   , partition.Value.TotalNumberOfMessagesProduced, now);

                    LibrdKafkaSensor.ScopedLibrdKafkaSensor.Record(PartitionTotalNumberOfBytesProducedSensor
                                                                   .Scoped(
                                                                       (LibrdKafkaBaseMetrics.TOPIC_TAG, topic.Value.TopicName),
                                                                       (LibrdKafkaBaseMetrics.BROKER_ID_TAG, partition.Value.BrokerId.ToString()),
                                                                       (LibrdKafkaBaseMetrics.PARTITION_ID_TAG, partition.Value.PartitionId.ToString()))
                                                                   , partition.Value.TotalNumberOfBytesProduced, now);

                    LibrdKafkaSensor.ScopedLibrdKafkaSensor.Record(PartitionNumberOfMessagesInFlightSensor
                                                                   .Scoped(
                                                                       (LibrdKafkaBaseMetrics.TOPIC_TAG, topic.Value.TopicName),
                                                                       (LibrdKafkaBaseMetrics.BROKER_ID_TAG, partition.Value.BrokerId.ToString()),
                                                                       (LibrdKafkaBaseMetrics.PARTITION_ID_TAG, partition.Value.PartitionId.ToString()))
                                                                   , partition.Value.NumberOfMessagesInFlight, now);

                    LibrdKafkaSensor.ScopedLibrdKafkaSensor.Record(PartitionNextExpectedAckSequenceSensor
                                                                   .Scoped(
                                                                       (LibrdKafkaBaseMetrics.TOPIC_TAG, topic.Value.TopicName),
                                                                       (LibrdKafkaBaseMetrics.BROKER_ID_TAG, partition.Value.BrokerId.ToString()),
                                                                       (LibrdKafkaBaseMetrics.PARTITION_ID_TAG, partition.Value.PartitionId.ToString()))
                                                                   , partition.Value.NextExpectedAckSequence, now);

                    LibrdKafkaSensor.ScopedLibrdKafkaSensor.Record(PartitionLastInternalMessageIdAckedSensor
                                                                   .Scoped(
                                                                       (LibrdKafkaBaseMetrics.TOPIC_TAG, topic.Value.TopicName),
                                                                       (LibrdKafkaBaseMetrics.BROKER_ID_TAG, partition.Value.BrokerId.ToString()),
                                                                       (LibrdKafkaBaseMetrics.PARTITION_ID_TAG, partition.Value.PartitionId.ToString()))
                                                                   , partition.Value.LastInternalMessageIdAcked, now);
                }

                BatchSizeAverageBytesSensor.RemoveOldScopeSensor(now);
                BatchMessageCountsAverageSensor.RemoveOldScopeSensor(now);
                PartitionTotalNumberOfMessagesProducedSensor.RemoveOldScopeSensor(now);
                PartitionTotalNumberOfBytesProducedSensor.RemoveOldScopeSensor(now);
                PartitionNumberOfMessagesInFlightSensor.RemoveOldScopeSensor(now);
                PartitionNextExpectedAckSequenceSensor.RemoveOldScopeSensor(now);
                PartitionLastInternalMessageIdAckedSensor.RemoveOldScopeSensor(now);
            }
        }
Ejemplo n.º 2
0
        private void PublishTopicsStats(Dictionary <string, TopicStatistic> statisticsTopics)
        {
            long now = DateTime.Now.GetMilliseconds();

            PublishTopicsStatistics(statisticsTopics);

            foreach (var topic in statisticsTopics)
            {
                foreach (var partition in topic.Value.Partitions)
                {
                    LibrdKafkaSensor.ScopedLibrdKafkaSensor.Record(ConsumerLagSensor
                                                                   .Scoped(
                                                                       (LibrdKafkaBaseMetrics.TOPIC_TAG, topic.Value.TopicName),
                                                                       (LibrdKafkaBaseMetrics.BROKER_ID_TAG, partition.Value.BrokerId.ToString()),
                                                                       (LibrdKafkaBaseMetrics.PARTITION_ID_TAG, partition.Value.PartitionId.ToString()))
                                                                   , partition.Value.ConsumerLag, now);

                    LibrdKafkaSensor.ScopedLibrdKafkaSensor.Record(TotalNumberOfMessagesConsumedByPartitionSensor
                                                                   .Scoped(
                                                                       (LibrdKafkaBaseMetrics.TOPIC_TAG, topic.Value.TopicName),
                                                                       (LibrdKafkaBaseMetrics.BROKER_ID_TAG, partition.Value.BrokerId.ToString()),
                                                                       (LibrdKafkaBaseMetrics.PARTITION_ID_TAG, partition.Value.PartitionId.ToString()))
                                                                   , partition.Value.TotalNumberOfMessagesconsumed, now);

                    LibrdKafkaSensor.ScopedLibrdKafkaSensor.Record(TotalNumberOfBytesConsumedByPartitionSensor
                                                                   .Scoped(
                                                                       (LibrdKafkaBaseMetrics.TOPIC_TAG, topic.Value.TopicName),
                                                                       (LibrdKafkaBaseMetrics.BROKER_ID_TAG, partition.Value.BrokerId.ToString()),
                                                                       (LibrdKafkaBaseMetrics.PARTITION_ID_TAG, partition.Value.PartitionId.ToString()))
                                                                   , partition.Value.TotalNumberOfBytesConsumed, now);
                }
            }

            BatchSizeAverageBytesSensor.RemoveOldScopeSensor(now);
            BatchMessageCountsAverageSensor.RemoveOldScopeSensor(now);
            ConsumerLagSensor.RemoveOldScopeSensor(now);
            TotalNumberOfMessagesConsumedByPartitionSensor.RemoveOldScopeSensor(now);
            TotalNumberOfBytesConsumedByPartitionSensor.RemoveOldScopeSensor(now);
        }
 private void ApplyTaskIdTagIfNeed()
 {
     if (!string.IsNullOrEmpty(taskId))
     {
         TotalNumberOfMessagesProducedSensor.ChangeTagValue(StreamMetricsRegistry.TASK_ID_TAG, taskId);
         TotalNumberOfMessageBytesProducedSensor.ChangeTagValue(StreamMetricsRegistry.TASK_ID_TAG, taskId);
         NumberOfOpsWaitinInQueueSensor.ChangeTagValue(StreamMetricsRegistry.TASK_ID_TAG, taskId);
         CurrentNumberOfMessagesInProducerQueuesSensor.ChangeTagValue(StreamMetricsRegistry.TASK_ID_TAG, taskId);
         CurrentSizeOfMessagesInProducerQueuesSensor.ChangeTagValue(StreamMetricsRegistry.TASK_ID_TAG, taskId);
         MaxMessagesAllowedOnProducerQueuesSensor.ChangeTagValue(StreamMetricsRegistry.TASK_ID_TAG, taskId);
         MaxSizeOfMessagesAllowedOnProducerQueuesSensor.ChangeTagValue(StreamMetricsRegistry.TASK_ID_TAG,
                                                                       taskId);
         TotalNumberOfRequestSentToKafkaSensor.ChangeTagValue(StreamMetricsRegistry.TASK_ID_TAG, taskId);
         TotalNumberOfBytesTransmittedToKafkaSensor.ChangeTagValue(StreamMetricsRegistry.TASK_ID_TAG, taskId);
         NumberOfRequestAwaitingTransmissionSensor.ChangeTagValue(StreamMetricsRegistry.TASK_ID_TAG, taskId);
         NumberOfMessagesAwaitingTransmissionSensor.ChangeTagValue(StreamMetricsRegistry.TASK_ID_TAG, taskId);
         NumberOfRequestInFlightSensor.ChangeTagValue(StreamMetricsRegistry.TASK_ID_TAG, taskId);
         NumberOfMessagesInFlightSensor.ChangeTagValue(StreamMetricsRegistry.TASK_ID_TAG, taskId);
         TotalNumberOfRequestSentSensor.ChangeTagValue(StreamMetricsRegistry.TASK_ID_TAG, taskId);
         TotalNumberOfBytesSentSensor.ChangeTagValue(StreamMetricsRegistry.TASK_ID_TAG, taskId);
         TotalNumberOfTransmissionErrorsSensor.ChangeTagValue(StreamMetricsRegistry.TASK_ID_TAG, taskId);
         TotalNumberOfRequestRetriesSensor.ChangeTagValue(StreamMetricsRegistry.TASK_ID_TAG, taskId);
         TotalNumberOfRequestTimeoutSensor.ChangeTagValue(StreamMetricsRegistry.TASK_ID_TAG, taskId);
         NumberOfConnectionAttempsSensor.ChangeTagValue(StreamMetricsRegistry.TASK_ID_TAG,
                                                        taskId); // Including successful, failed and name resolution failures
         NumberOfDisconnectsSensor.ChangeTagValue(StreamMetricsRegistry.TASK_ID_TAG, taskId);
         InternalQueueProducerLatencyAverageMsSensor.ChangeTagValue(StreamMetricsRegistry.TASK_ID_TAG, taskId);
         InternalRequestQueueLatencyAverageMsSensor.ChangeTagValue(StreamMetricsRegistry.TASK_ID_TAG, taskId);
         BrokerLatencyAverageMsSensor.ChangeTagValue(StreamMetricsRegistry.TASK_ID_TAG, taskId);
         BatchSizeAverageBytesSensor.ChangeTagValue(StreamMetricsRegistry.TASK_ID_TAG, taskId);
         BatchMessageCountsAverageSensor.ChangeTagValue(StreamMetricsRegistry.TASK_ID_TAG, taskId);
         PartitionTotalNumberOfMessagesProducedSensor.ChangeTagValue(StreamMetricsRegistry.TASK_ID_TAG, taskId);
         PartitionTotalNumberOfBytesProducedSensor.ChangeTagValue(StreamMetricsRegistry.TASK_ID_TAG, taskId);
         PartitionNumberOfMessagesInFlightSensor.ChangeTagValue(StreamMetricsRegistry.TASK_ID_TAG, taskId);
         PartitionNextExpectedAckSequenceSensor.ChangeTagValue(StreamMetricsRegistry.TASK_ID_TAG, taskId);
         PartitionLastInternalMessageIdAckedSensor.ChangeTagValue(StreamMetricsRegistry.TASK_ID_TAG, taskId);
     }
 }