public void TestRegisterConsumerMetrics() { var consumerStatisticsHandler = new ConsumerStatisticsHandler( config.ClientId, config.ApplicationId, threadId); consumerStatisticsHandler.Register(streamMetricsRegistry); var sensorLibrdkafkaConsumerMetrics = streamMetricsRegistry .GetThreadScopeSensor(threadId) .Count(s => s.Name.StartsWith($"{StreamMetricsRegistry.SENSOR_LIBRDKAFKA_LABEL}.")); Assert.IsTrue(sensorLibrdkafkaConsumerMetrics > 0); var statistics = JsonConvert.DeserializeObject <Statistics>(jsonPayload); consumerStatisticsHandler.Publish(statistics); }
public void ProcessorNodeMetricsTest() { var serdes = new StringSerDes(); var rd = new Random(); int nbMessage = rd.Next(0, 4000); // simulate processing message for (int i = 0; i < nbMessage; ++i) { string key = "key" + i, value = "value" + i; var r = new ConsumeResult <byte[], byte[]>() { Message = new Message <byte[], byte[]>() { Key = serdes.Serialize(key, new SerializationContext()), Value = serdes.Serialize(value, new SerializationContext()), Timestamp = new Timestamp(DateTime.Now) }, Offset = i, IsPartitionEOF = false, Topic = topicPartition.Topic, Partition = topicPartition.Partition }; context.SetRecordMetaData(r); sourceProcessor.Process(key, value); } long now = DateTime.Now.GetMilliseconds(); var sensors = streamMetricsRegistry.GetThreadScopeSensor(threadId); var processorSensor = sourceProcessor.ProcessSensor; Assert.AreEqual(2, processorSensor.Metrics.Count()); Assert.AreEqual(nbMessage, processorSensor.Metrics[MetricName.NameAndGroup( ProcessorNodeMetrics.PROCESS + StreamMetricsRegistry.TOTAL_SUFFIX, StreamMetricsRegistry.PROCESSOR_NODE_LEVEL_GROUP)].Value); Assert.IsTrue( (double)processorSensor.Metrics[MetricName.NameAndGroup( ProcessorNodeMetrics.PROCESS + StreamMetricsRegistry.RATE_SUFFIX, StreamMetricsRegistry.PROCESSOR_NODE_LEVEL_GROUP)].Value > 0d); }
public void ThreadMetricsTest() { var serdes = new StringSerDes(); var cloneConfig = config.Clone(); cloneConfig.ApplicationId = "consume-test"; var producer = mockKafkaSupplier.GetProducer(cloneConfig.ToProducerConfig()); var consumer = mockKafkaSupplier.GetConsumer(cloneConfig.ToConsumerConfig("test-consum"), null); consumer.Subscribe("topic2"); thread.Start(token.Token); AssertExtensions.WaitUntil(() => thread.ActiveTasks.Count() == numberPartitions, TimeSpan.FromSeconds(5), TimeSpan.FromMilliseconds(100)); int nbMessage = 1000; // produce ${nbMessage} messages to input topic for (int i = 0; i < nbMessage; ++i) { producer.Produce("topic", new Confluent.Kafka.Message <byte[], byte[]> { Key = serdes.Serialize("key" + i, new SerializationContext()), Value = serdes.Serialize("Hi" + i, new SerializationContext()) }); } var messagesSink = new List <ConsumeResult <byte[], byte[]> >(); AssertExtensions.WaitUntil(() => { messagesSink.AddRange(consumer.ConsumeRecords(TimeSpan.FromSeconds(1))); return(messagesSink.Count == nbMessage); }, TimeSpan.FromSeconds(5), TimeSpan.FromMilliseconds(10)); // waiting end of processing Thread.Sleep(1000); long now = DateTime.Now.GetMilliseconds(); var sensors = streamMetricsRegistry.GetThreadScopeSensor(threadId); var createTaskSensor = sensors.FirstOrDefault(s => s.Name.Equals(GetSensorName(ThreadMetrics.CREATE_TASK))); Assert.AreEqual(2, createTaskSensor.Metrics.Count()); Assert.AreEqual(numberPartitions, createTaskSensor.Metrics[MetricName.NameAndGroup( ThreadMetrics.CREATE_TASK + StreamMetricsRegistry.TOTAL_SUFFIX, StreamMetricsRegistry.THREAD_LEVEL_GROUP)].Value); Assert.IsTrue( (double)(createTaskSensor.Metrics[MetricName.NameAndGroup( ThreadMetrics.CREATE_TASK + StreamMetricsRegistry.RATE_SUFFIX, StreamMetricsRegistry.THREAD_LEVEL_GROUP)].Value) > 0d); var closeTaskSensor = sensors.FirstOrDefault(s => s.Name.Equals(GetSensorName(ThreadMetrics.CLOSE_TASK))); Assert.AreEqual(2, closeTaskSensor.Metrics.Count()); Assert.AreEqual(0d, closeTaskSensor.Metrics[MetricName.NameAndGroup( ThreadMetrics.CLOSE_TASK + StreamMetricsRegistry.TOTAL_SUFFIX, StreamMetricsRegistry.THREAD_LEVEL_GROUP)].Value); Assert.AreEqual(0d, closeTaskSensor.Metrics[MetricName.NameAndGroup( ThreadMetrics.CLOSE_TASK + StreamMetricsRegistry.RATE_SUFFIX, StreamMetricsRegistry.THREAD_LEVEL_GROUP)].Value); var commitSensor = sensors.FirstOrDefault(s => s.Name.Equals(GetSensorName(ThreadMetrics.COMMIT))); Assert.AreEqual(4, commitSensor.Metrics.Count()); Assert.IsTrue( (double)commitSensor.Metrics[MetricName.NameAndGroup( ThreadMetrics.COMMIT + StreamMetricsRegistry.TOTAL_SUFFIX, StreamMetricsRegistry.THREAD_LEVEL_GROUP)].Value > 0d); Assert.IsTrue( (double)commitSensor.Metrics[MetricName.NameAndGroup( ThreadMetrics.COMMIT + StreamMetricsRegistry.RATE_SUFFIX, StreamMetricsRegistry.THREAD_LEVEL_GROUP)].Value > 0d); Assert.IsTrue( (double)commitSensor.Metrics[MetricName.NameAndGroup( ThreadMetrics.COMMIT + StreamMetricsRegistry.LATENCY_SUFFIX + StreamMetricsRegistry.AVG_SUFFIX, StreamMetricsRegistry.THREAD_LEVEL_GROUP)].Value > 0d); Assert.IsTrue( (double)commitSensor.Metrics[MetricName.NameAndGroup( ThreadMetrics.COMMIT + StreamMetricsRegistry.LATENCY_SUFFIX + StreamMetricsRegistry.MAX_SUFFIX, StreamMetricsRegistry.THREAD_LEVEL_GROUP)].Value > 0d); var pollSensor = sensors.FirstOrDefault(s => s.Name.Equals(GetSensorName(ThreadMetrics.POLL))); Assert.AreEqual(4, pollSensor.Metrics.Count()); Assert.IsTrue( (double)pollSensor.Metrics[MetricName.NameAndGroup( ThreadMetrics.POLL + StreamMetricsRegistry.TOTAL_SUFFIX, StreamMetricsRegistry.THREAD_LEVEL_GROUP)].Value > 0d); Assert.IsTrue( (double)pollSensor.Metrics[MetricName.NameAndGroup( ThreadMetrics.POLL + StreamMetricsRegistry.RATE_SUFFIX, StreamMetricsRegistry.THREAD_LEVEL_GROUP)].Value > 0d); Assert.IsTrue( (double)pollSensor.Metrics[MetricName.NameAndGroup( ThreadMetrics.POLL + StreamMetricsRegistry.LATENCY_SUFFIX + StreamMetricsRegistry.AVG_SUFFIX, StreamMetricsRegistry.THREAD_LEVEL_GROUP)].Value > 0d); Assert.IsTrue( (double)pollSensor.Metrics[MetricName.NameAndGroup( ThreadMetrics.POLL + StreamMetricsRegistry.LATENCY_SUFFIX + StreamMetricsRegistry.MAX_SUFFIX, StreamMetricsRegistry.THREAD_LEVEL_GROUP)].Value > 0d); var pollRecordsSensor = sensors.FirstOrDefault(s => s.Name.Equals(GetSensorName(ThreadMetrics.POLL + StreamMetricsRegistry.RECORDS_SUFFIX))); Assert.AreEqual(2, pollRecordsSensor.Metrics.Count()); Assert.IsTrue( (double)pollRecordsSensor.Metrics[MetricName.NameAndGroup( ThreadMetrics.POLL + StreamMetricsRegistry.RECORDS_SUFFIX + StreamMetricsRegistry.AVG_SUFFIX, StreamMetricsRegistry.THREAD_LEVEL_GROUP)].Value > 0d); Assert.IsTrue( (double)pollRecordsSensor.Metrics[MetricName.NameAndGroup( ThreadMetrics.POLL + StreamMetricsRegistry.RECORDS_SUFFIX + StreamMetricsRegistry.MAX_SUFFIX, StreamMetricsRegistry.THREAD_LEVEL_GROUP)].Value > 0d); var processRecordsSensor = sensors.FirstOrDefault(s => s.Name.Equals(GetSensorName(ThreadMetrics.PROCESS + StreamMetricsRegistry.RECORDS_SUFFIX))); Assert.AreEqual(2, processRecordsSensor.Metrics.Count()); Assert.IsTrue( (double)processRecordsSensor.Metrics[MetricName.NameAndGroup( ThreadMetrics.PROCESS + StreamMetricsRegistry.RECORDS_SUFFIX + StreamMetricsRegistry.AVG_SUFFIX, StreamMetricsRegistry.THREAD_LEVEL_GROUP)].Value > 0d); Assert.IsTrue( (double)processRecordsSensor.Metrics[MetricName.NameAndGroup( ThreadMetrics.PROCESS + StreamMetricsRegistry.RECORDS_SUFFIX + StreamMetricsRegistry.MAX_SUFFIX, StreamMetricsRegistry.THREAD_LEVEL_GROUP)].Value > 0d); var processRateSensor = sensors.FirstOrDefault(s => s.Name.Equals(GetSensorName(ThreadMetrics.PROCESS + StreamMetricsRegistry.RATE_SUFFIX))); Assert.AreEqual(2, processRateSensor.Metrics.Count()); Assert.IsTrue( (double)processRateSensor.Metrics[MetricName.NameAndGroup( ThreadMetrics.PROCESS + StreamMetricsRegistry.RATE_SUFFIX + StreamMetricsRegistry.RATE_SUFFIX, StreamMetricsRegistry.THREAD_LEVEL_GROUP)].Value > 0d); Assert.IsTrue( (double)processRateSensor.Metrics[MetricName.NameAndGroup( ThreadMetrics.PROCESS + StreamMetricsRegistry.RATE_SUFFIX + StreamMetricsRegistry.TOTAL_SUFFIX, StreamMetricsRegistry.THREAD_LEVEL_GROUP)].Value > 0d); var processLatencySensor = sensors.FirstOrDefault(s => s.Name.Equals(GetSensorName(ThreadMetrics.PROCESS + StreamMetricsRegistry.LATENCY_SUFFIX))); Assert.AreEqual(2, processLatencySensor.Metrics.Count()); Assert.IsTrue( (double)processLatencySensor.Metrics[MetricName.NameAndGroup( ThreadMetrics.PROCESS + StreamMetricsRegistry.LATENCY_SUFFIX + StreamMetricsRegistry.AVG_SUFFIX, StreamMetricsRegistry.THREAD_LEVEL_GROUP)].Value > 0d); Assert.IsTrue( (double)processLatencySensor.Metrics[MetricName.NameAndGroup( ThreadMetrics.PROCESS + StreamMetricsRegistry.LATENCY_SUFFIX + StreamMetricsRegistry.MAX_SUFFIX, StreamMetricsRegistry.THREAD_LEVEL_GROUP)].Value > 0d); // ratio sensors var processRatioSensor = sensors.FirstOrDefault(s => s.Name.Equals(GetSensorName(ThreadMetrics.PROCESS + StreamMetricsRegistry.RATIO_SUFFIX))); var pollRatioSensor = sensors.FirstOrDefault(s => s.Name.Equals(GetSensorName(ThreadMetrics.POLL + StreamMetricsRegistry.RATIO_SUFFIX))); var commitRatioSensor = sensors.FirstOrDefault(s => s.Name.Equals(GetSensorName(ThreadMetrics.COMMIT + StreamMetricsRegistry.RATIO_SUFFIX))); Assert.AreEqual(1, processRatioSensor.Metrics.Count()); Assert.AreEqual(1, pollRatioSensor.Metrics.Count()); Assert.AreEqual(1, commitRatioSensor.Metrics.Count()); var processRatioValue = (double)processRatioSensor.Metrics[MetricName.NameAndGroup( ThreadMetrics.PROCESS + StreamMetricsRegistry.RATIO_SUFFIX, StreamMetricsRegistry.THREAD_LEVEL_GROUP)].Value; var pollRatioValue = (double)pollRatioSensor.Metrics[MetricName.NameAndGroup( ThreadMetrics.POLL + StreamMetricsRegistry.RATIO_SUFFIX, StreamMetricsRegistry.THREAD_LEVEL_GROUP)].Value; var commitRatioValue = (double)commitRatioSensor.Metrics[MetricName.NameAndGroup( ThreadMetrics.COMMIT + StreamMetricsRegistry.RATIO_SUFFIX, StreamMetricsRegistry.THREAD_LEVEL_GROUP)].Value; double total = Math.Round(processRatioValue + pollRatioValue + commitRatioValue, 2); // we accept 10% of lost Assert.IsTrue(total >= 0.90d); }
private void ExportMetrics(long now) { var sensors = streamMetricsRegistry.GetThreadScopeSensor(threadId); Task.Factory.StartNew(() => streamConfig.MetricsReporter?.Invoke(sensors)); }
public void TaskMetricsTest() { var serdes = new StringSerDes(); var cloneConfig = config.Clone(); cloneConfig.ApplicationId = "consume-test"; var producer = syncKafkaSupplier.GetProducer(cloneConfig.ToProducerConfig()); var consumer = syncKafkaSupplier.GetConsumer(cloneConfig.ToConsumerConfig("test-consum"), null); consumer.Subscribe("topic2"); int nbMessage = 1000; // produce 1000 messages to input topic List <ConsumeResult <byte[], byte[]> > messages = new List <ConsumeResult <byte[], byte[]> >(); int offset = 0; for (int i = 0; i < nbMessage; ++i) { messages.Add( new ConsumeResult <byte[], byte[]> { Message = new Message <byte[], byte[]> { Key = serdes.Serialize($"key{i + 1}", new SerializationContext()), Value = serdes.Serialize($"value{i + 1}", new SerializationContext()) }, TopicPartitionOffset = new TopicPartitionOffset(topicPartition, offset++) }); } task.AddRecords(messages); while (task.CanProcess(DateTime.Now.GetMilliseconds())) { Assert.IsTrue(task.Process()); Assert.IsTrue(task.CommitNeeded); task.Commit(); } var messagesSink = new List <ConsumeResult <byte[], byte[]> >(); AssertExtensions.WaitUntil(() => { messagesSink.AddRange(consumer.ConsumeRecords(TimeSpan.FromSeconds(1))); return(messagesSink.Count < nbMessage); }, TimeSpan.FromSeconds(5), TimeSpan.FromMilliseconds(10)); long now = DateTime.Now.GetMilliseconds(); var sensors = streamMetricsRegistry.GetThreadScopeSensor(threadId); var processorSensor = sensors.FirstOrDefault(s => s.Name.Equals(GetSensorName(TaskMetrics.PROCESS))); Assert.AreEqual(2, processorSensor.Metrics.Count()); Assert.AreEqual(nbMessage, processorSensor.Metrics[MetricName.NameAndGroup( TaskMetrics.PROCESS + StreamMetricsRegistry.TOTAL_SUFFIX, StreamMetricsRegistry.TASK_LEVEL_GROUP)].Value); Assert.IsTrue( (double)(processorSensor.Metrics[MetricName.NameAndGroup( TaskMetrics.PROCESS + StreamMetricsRegistry.RATE_SUFFIX, StreamMetricsRegistry.TASK_LEVEL_GROUP)].Value) > 0d); var enforcedProcessorSensor = sensors.FirstOrDefault(s => s.Name.Equals(GetSensorName(TaskMetrics.ENFORCED_PROCESSING))); Assert.AreEqual(2, enforcedProcessorSensor.Metrics.Count()); Assert.AreEqual(0, enforcedProcessorSensor.Metrics[MetricName.NameAndGroup( TaskMetrics.ENFORCED_PROCESSING + StreamMetricsRegistry.TOTAL_SUFFIX, StreamMetricsRegistry.TASK_LEVEL_GROUP)].Value); Assert.AreEqual(0, enforcedProcessorSensor.Metrics[MetricName.NameAndGroup( TaskMetrics.ENFORCED_PROCESSING + StreamMetricsRegistry.RATE_SUFFIX, StreamMetricsRegistry.TASK_LEVEL_GROUP)].Value); var processLatency = sensors.FirstOrDefault(s => s.Name.Equals(GetSensorName(TaskMetrics.PROCESS_LATENCY))); Assert.AreEqual(2, processLatency.Metrics.Count()); Assert.IsTrue( (double)processLatency.Metrics[MetricName.NameAndGroup( TaskMetrics.PROCESS_LATENCY + StreamMetricsRegistry.AVG_SUFFIX, StreamMetricsRegistry.TASK_LEVEL_GROUP)].Value > 0d); Assert.IsTrue( (double)processLatency.Metrics[MetricName.NameAndGroup( TaskMetrics.PROCESS_LATENCY + StreamMetricsRegistry.MAX_SUFFIX, StreamMetricsRegistry.TASK_LEVEL_GROUP)].Value > 0d); var commitSensor = sensors.FirstOrDefault(s => s.Name.Equals(GetSensorName(TaskMetrics.COMMIT))); Assert.AreEqual(2, commitSensor.Metrics.Count()); Assert.AreEqual(nbMessage, commitSensor.Metrics[MetricName.NameAndGroup( TaskMetrics.COMMIT + StreamMetricsRegistry.TOTAL_SUFFIX, StreamMetricsRegistry.TASK_LEVEL_GROUP)].Value); Assert.IsTrue( (double)commitSensor.Metrics[MetricName.NameAndGroup( TaskMetrics.COMMIT + StreamMetricsRegistry.RATE_SUFFIX, StreamMetricsRegistry.TASK_LEVEL_GROUP)].Value > 0d); var droppedRecordSensor = sensors.FirstOrDefault(s => s.Name.Equals(GetSensorName(TaskMetrics.DROPPED_RECORDS))); Assert.AreEqual(2, droppedRecordSensor.Metrics.Count()); Assert.AreEqual(0, droppedRecordSensor.Metrics[MetricName.NameAndGroup( TaskMetrics.DROPPED_RECORDS + StreamMetricsRegistry.TOTAL_SUFFIX, StreamMetricsRegistry.TASK_LEVEL_GROUP)].Value); Assert.AreEqual(0, droppedRecordSensor.Metrics[MetricName.NameAndGroup( TaskMetrics.DROPPED_RECORDS + StreamMetricsRegistry.RATE_SUFFIX, StreamMetricsRegistry.TASK_LEVEL_GROUP)].Value); var activeBufferedRecordSensor = sensors.FirstOrDefault(s => s.Name.Equals(GetSensorName(TaskMetrics.ACTIVE_TASK_PREFIX + TaskMetrics.BUFFER_COUNT))); Assert.AreEqual(1, activeBufferedRecordSensor.Metrics.Count()); Assert.AreEqual(0, activeBufferedRecordSensor.Metrics[MetricName.NameAndGroup( TaskMetrics.ACTIVE_TASK_PREFIX + TaskMetrics.BUFFER_COUNT, StreamMetricsRegistry.TASK_LEVEL_GROUP)].Value); var restorationRecordsSensor = sensors.FirstOrDefault(s => s.Name.Equals(GetSensorName(TaskMetrics.RESTORATION_RECORDS))); Assert.AreEqual(1, restorationRecordsSensor.Metrics.Count()); Assert.AreEqual(0, restorationRecordsSensor.Metrics[MetricName.NameAndGroup( TaskMetrics.RESTORATION_RECORDS, StreamMetricsRegistry.TASK_LEVEL_GROUP)].Value); var activeRestorationSensor = sensors.FirstOrDefault(s => s.Name.Equals(GetSensorName(TaskMetrics.ACTIVE_RESTORATION))); Assert.AreEqual(1, activeRestorationSensor.Metrics.Count()); Assert.AreEqual(0, activeRestorationSensor.Metrics[MetricName.NameAndGroup( TaskMetrics.ACTIVE_RESTORATION, StreamMetricsRegistry.TASK_LEVEL_GROUP)].Value); }