private void PublishTopicsStats(Dictionary <string, TopicStatistic> statisticsTopics)
{
long now = DateTime.Now.GetMilliseconds();
PublishTopicsStatistics(statisticsTopics);
foreach (var topic in statisticsTopics)
{
foreach (var partition in topic.Value.Partitions)
{
LibrdKafkaSensor.ScopedLibrdKafkaSensor.Record(PartitionTotalNumberOfMessagesProducedSensor
.Scoped(
(LibrdKafkaBaseMetrics.TOPIC_TAG, topic.Value.TopicName),
(LibrdKafkaBaseMetrics.BROKER_ID_TAG, partition.Value.BrokerId.ToString()),
(LibrdKafkaBaseMetrics.PARTITION_ID_TAG, partition.Value.PartitionId.ToString()))
, partition.Value.TotalNumberOfMessagesProduced, now);
LibrdKafkaSensor.ScopedLibrdKafkaSensor.Record(PartitionTotalNumberOfBytesProducedSensor
.Scoped(
(LibrdKafkaBaseMetrics.TOPIC_TAG, topic.Value.TopicName),
(LibrdKafkaBaseMetrics.BROKER_ID_TAG, partition.Value.BrokerId.ToString()),
(LibrdKafkaBaseMetrics.PARTITION_ID_TAG, partition.Value.PartitionId.ToString()))
, partition.Value.TotalNumberOfBytesProduced, now);
LibrdKafkaSensor.ScopedLibrdKafkaSensor.Record(PartitionNumberOfMessagesInFlightSensor
.Scoped(
(LibrdKafkaBaseMetrics.TOPIC_TAG, topic.Value.TopicName),
(LibrdKafkaBaseMetrics.BROKER_ID_TAG, partition.Value.BrokerId.ToString()),
(LibrdKafkaBaseMetrics.PARTITION_ID_TAG, partition.Value.PartitionId.ToString()))
, partition.Value.NumberOfMessagesInFlight, now);
LibrdKafkaSensor.ScopedLibrdKafkaSensor.Record(PartitionNextExpectedAckSequenceSensor
.Scoped(
(LibrdKafkaBaseMetrics.TOPIC_TAG, topic.Value.TopicName),
(LibrdKafkaBaseMetrics.BROKER_ID_TAG, partition.Value.BrokerId.ToString()),
(LibrdKafkaBaseMetrics.PARTITION_ID_TAG, partition.Value.PartitionId.ToString()))
, partition.Value.NextExpectedAckSequence, now);
LibrdKafkaSensor.ScopedLibrdKafkaSensor.Record(PartitionLastInternalMessageIdAckedSensor
.Scoped(
(LibrdKafkaBaseMetrics.TOPIC_TAG, topic.Value.TopicName),
(LibrdKafkaBaseMetrics.BROKER_ID_TAG, partition.Value.BrokerId.ToString()),
(LibrdKafkaBaseMetrics.PARTITION_ID_TAG, partition.Value.PartitionId.ToString()))
, partition.Value.LastInternalMessageIdAcked, now);
}
BatchSizeAverageBytesSensor.RemoveOldScopeSensor(now);
BatchMessageCountsAverageSensor.RemoveOldScopeSensor(now);
PartitionTotalNumberOfMessagesProducedSensor.RemoveOldScopeSensor(now);
PartitionTotalNumberOfBytesProducedSensor.RemoveOldScopeSensor(now);
PartitionNumberOfMessagesInFlightSensor.RemoveOldScopeSensor(now);
PartitionNextExpectedAckSequenceSensor.RemoveOldScopeSensor(now);
PartitionLastInternalMessageIdAckedSensor.RemoveOldScopeSensor(now);
}
}
private void PublishTopicsStats(Dictionary <string, TopicStatistic> statisticsTopics)
{
long now = DateTime.Now.GetMilliseconds();
PublishTopicsStatistics(statisticsTopics);
foreach (var topic in statisticsTopics)
{
foreach (var partition in topic.Value.Partitions)
{
LibrdKafkaSensor.ScopedLibrdKafkaSensor.Record(ConsumerLagSensor
.Scoped(
(LibrdKafkaBaseMetrics.TOPIC_TAG, topic.Value.TopicName),
(LibrdKafkaBaseMetrics.BROKER_ID_TAG, partition.Value.BrokerId.ToString()),
(LibrdKafkaBaseMetrics.PARTITION_ID_TAG, partition.Value.PartitionId.ToString()))
, partition.Value.ConsumerLag, now);
LibrdKafkaSensor.ScopedLibrdKafkaSensor.Record(TotalNumberOfMessagesConsumedByPartitionSensor
.Scoped(
(LibrdKafkaBaseMetrics.TOPIC_TAG, topic.Value.TopicName),
(LibrdKafkaBaseMetrics.BROKER_ID_TAG, partition.Value.BrokerId.ToString()),
(LibrdKafkaBaseMetrics.PARTITION_ID_TAG, partition.Value.PartitionId.ToString()))
, partition.Value.TotalNumberOfMessagesconsumed, now);
LibrdKafkaSensor.ScopedLibrdKafkaSensor.Record(TotalNumberOfBytesConsumedByPartitionSensor
.Scoped(
(LibrdKafkaBaseMetrics.TOPIC_TAG, topic.Value.TopicName),
(LibrdKafkaBaseMetrics.BROKER_ID_TAG, partition.Value.BrokerId.ToString()),
(LibrdKafkaBaseMetrics.PARTITION_ID_TAG, partition.Value.PartitionId.ToString()))
, partition.Value.TotalNumberOfBytesConsumed, now);
}
}
BatchSizeAverageBytesSensor.RemoveOldScopeSensor(now);
BatchMessageCountsAverageSensor.RemoveOldScopeSensor(now);
ConsumerLagSensor.RemoveOldScopeSensor(now);
TotalNumberOfMessagesConsumedByPartitionSensor.RemoveOldScopeSensor(now);
TotalNumberOfBytesConsumedByPartitionSensor.RemoveOldScopeSensor(now);
}
private void ApplyTaskIdTagIfNeed()
{
if (!string.IsNullOrEmpty(taskId))
{
TotalNumberOfMessagesProducedSensor.ChangeTagValue(StreamMetricsRegistry.TASK_ID_TAG, taskId);
TotalNumberOfMessageBytesProducedSensor.ChangeTagValue(StreamMetricsRegistry.TASK_ID_TAG, taskId);
NumberOfOpsWaitinInQueueSensor.ChangeTagValue(StreamMetricsRegistry.TASK_ID_TAG, taskId);
CurrentNumberOfMessagesInProducerQueuesSensor.ChangeTagValue(StreamMetricsRegistry.TASK_ID_TAG, taskId);
CurrentSizeOfMessagesInProducerQueuesSensor.ChangeTagValue(StreamMetricsRegistry.TASK_ID_TAG, taskId);
MaxMessagesAllowedOnProducerQueuesSensor.ChangeTagValue(StreamMetricsRegistry.TASK_ID_TAG, taskId);
MaxSizeOfMessagesAllowedOnProducerQueuesSensor.ChangeTagValue(StreamMetricsRegistry.TASK_ID_TAG,
taskId);
TotalNumberOfRequestSentToKafkaSensor.ChangeTagValue(StreamMetricsRegistry.TASK_ID_TAG, taskId);
TotalNumberOfBytesTransmittedToKafkaSensor.ChangeTagValue(StreamMetricsRegistry.TASK_ID_TAG, taskId);
NumberOfRequestAwaitingTransmissionSensor.ChangeTagValue(StreamMetricsRegistry.TASK_ID_TAG, taskId);
NumberOfMessagesAwaitingTransmissionSensor.ChangeTagValue(StreamMetricsRegistry.TASK_ID_TAG, taskId);
NumberOfRequestInFlightSensor.ChangeTagValue(StreamMetricsRegistry.TASK_ID_TAG, taskId);
NumberOfMessagesInFlightSensor.ChangeTagValue(StreamMetricsRegistry.TASK_ID_TAG, taskId);
TotalNumberOfRequestSentSensor.ChangeTagValue(StreamMetricsRegistry.TASK_ID_TAG, taskId);
TotalNumberOfBytesSentSensor.ChangeTagValue(StreamMetricsRegistry.TASK_ID_TAG, taskId);
TotalNumberOfTransmissionErrorsSensor.ChangeTagValue(StreamMetricsRegistry.TASK_ID_TAG, taskId);
TotalNumberOfRequestRetriesSensor.ChangeTagValue(StreamMetricsRegistry.TASK_ID_TAG, taskId);
TotalNumberOfRequestTimeoutSensor.ChangeTagValue(StreamMetricsRegistry.TASK_ID_TAG, taskId);
NumberOfConnectionAttempsSensor.ChangeTagValue(StreamMetricsRegistry.TASK_ID_TAG,
taskId); // Including successful, failed and name resolution failures
NumberOfDisconnectsSensor.ChangeTagValue(StreamMetricsRegistry.TASK_ID_TAG, taskId);
InternalQueueProducerLatencyAverageMsSensor.ChangeTagValue(StreamMetricsRegistry.TASK_ID_TAG, taskId);
InternalRequestQueueLatencyAverageMsSensor.ChangeTagValue(StreamMetricsRegistry.TASK_ID_TAG, taskId);
BrokerLatencyAverageMsSensor.ChangeTagValue(StreamMetricsRegistry.TASK_ID_TAG, taskId);
BatchSizeAverageBytesSensor.ChangeTagValue(StreamMetricsRegistry.TASK_ID_TAG, taskId);
BatchMessageCountsAverageSensor.ChangeTagValue(StreamMetricsRegistry.TASK_ID_TAG, taskId);
PartitionTotalNumberOfMessagesProducedSensor.ChangeTagValue(StreamMetricsRegistry.TASK_ID_TAG, taskId);
PartitionTotalNumberOfBytesProducedSensor.ChangeTagValue(StreamMetricsRegistry.TASK_ID_TAG, taskId);
PartitionNumberOfMessagesInFlightSensor.ChangeTagValue(StreamMetricsRegistry.TASK_ID_TAG, taskId);
PartitionNextExpectedAckSequenceSensor.ChangeTagValue(StreamMetricsRegistry.TASK_ID_TAG, taskId);
PartitionLastInternalMessageIdAckedSensor.ChangeTagValue(StreamMetricsRegistry.TASK_ID_TAG, taskId);
}
}
