public void TestRegisterConsumerMetrics()
{
var consumerStatisticsHandler = new ConsumerStatisticsHandler(
config.ClientId,
config.ApplicationId,
threadId);
consumerStatisticsHandler.Register(streamMetricsRegistry);
var sensorLibrdkafkaConsumerMetrics = streamMetricsRegistry
.GetThreadScopeSensor(threadId)
.Count(s => s.Name.StartsWith($"{StreamMetricsRegistry.SENSOR_LIBRDKAFKA_LABEL}."));
Assert.IsTrue(sensorLibrdkafkaConsumerMetrics > 0);
var statistics = JsonConvert.DeserializeObject <Statistics>(jsonPayload);
consumerStatisticsHandler.Publish(statistics);
}
public void ProcessorNodeMetricsTest()
{
var serdes = new StringSerDes();
var rd = new Random();
int nbMessage = rd.Next(0, 4000);
// simulate processing message
for (int i = 0; i < nbMessage; ++i)
{
string key = "key" + i, value = "value" + i;
var r = new ConsumeResult <byte[], byte[]>()
{
Message = new Message <byte[], byte[]>()
{
Key = serdes.Serialize(key, new SerializationContext()),
Value = serdes.Serialize(value, new SerializationContext()),
Timestamp = new Timestamp(DateTime.Now)
},
Offset = i,
IsPartitionEOF = false,
Topic = topicPartition.Topic,
Partition = topicPartition.Partition
};
context.SetRecordMetaData(r);
sourceProcessor.Process(key, value);
}
long now = DateTime.Now.GetMilliseconds();
var sensors = streamMetricsRegistry.GetThreadScopeSensor(threadId);
var processorSensor = sourceProcessor.ProcessSensor;
Assert.AreEqual(2, processorSensor.Metrics.Count());
Assert.AreEqual(nbMessage,
processorSensor.Metrics[MetricName.NameAndGroup(
ProcessorNodeMetrics.PROCESS + StreamMetricsRegistry.TOTAL_SUFFIX,
StreamMetricsRegistry.PROCESSOR_NODE_LEVEL_GROUP)].Value);
Assert.IsTrue(
(double)processorSensor.Metrics[MetricName.NameAndGroup(
ProcessorNodeMetrics.PROCESS + StreamMetricsRegistry.RATE_SUFFIX,
StreamMetricsRegistry.PROCESSOR_NODE_LEVEL_GROUP)].Value > 0d);
}
public void ThreadMetricsTest()
{
var serdes = new StringSerDes();
var cloneConfig = config.Clone();
cloneConfig.ApplicationId = "consume-test";
var producer = mockKafkaSupplier.GetProducer(cloneConfig.ToProducerConfig());
var consumer = mockKafkaSupplier.GetConsumer(cloneConfig.ToConsumerConfig("test-consum"), null);
consumer.Subscribe("topic2");
thread.Start(token.Token);
AssertExtensions.WaitUntil(() => thread.ActiveTasks.Count() == numberPartitions,
TimeSpan.FromSeconds(5),
TimeSpan.FromMilliseconds(100));
int nbMessage = 1000;
// produce ${nbMessage} messages to input topic
for (int i = 0; i < nbMessage; ++i)
{
producer.Produce("topic", new Confluent.Kafka.Message <byte[], byte[]>
{
Key = serdes.Serialize("key" + i, new SerializationContext()),
Value = serdes.Serialize("Hi" + i, new SerializationContext())
});
}
var messagesSink = new List <ConsumeResult <byte[], byte[]> >();
AssertExtensions.WaitUntil(() =>
{
messagesSink.AddRange(consumer.ConsumeRecords(TimeSpan.FromSeconds(1)));
return(messagesSink.Count == nbMessage);
}, TimeSpan.FromSeconds(5),
TimeSpan.FromMilliseconds(10));
// waiting end of processing
Thread.Sleep(1000);
long now = DateTime.Now.GetMilliseconds();
var sensors = streamMetricsRegistry.GetThreadScopeSensor(threadId);
var createTaskSensor = sensors.FirstOrDefault(s => s.Name.Equals(GetSensorName(ThreadMetrics.CREATE_TASK)));
Assert.AreEqual(2, createTaskSensor.Metrics.Count());
Assert.AreEqual(numberPartitions,
createTaskSensor.Metrics[MetricName.NameAndGroup(
ThreadMetrics.CREATE_TASK + StreamMetricsRegistry.TOTAL_SUFFIX,
StreamMetricsRegistry.THREAD_LEVEL_GROUP)].Value);
Assert.IsTrue(
(double)(createTaskSensor.Metrics[MetricName.NameAndGroup(
ThreadMetrics.CREATE_TASK + StreamMetricsRegistry.RATE_SUFFIX,
StreamMetricsRegistry.THREAD_LEVEL_GROUP)].Value) > 0d);
var closeTaskSensor = sensors.FirstOrDefault(s => s.Name.Equals(GetSensorName(ThreadMetrics.CLOSE_TASK)));
Assert.AreEqual(2, closeTaskSensor.Metrics.Count());
Assert.AreEqual(0d,
closeTaskSensor.Metrics[MetricName.NameAndGroup(
ThreadMetrics.CLOSE_TASK + StreamMetricsRegistry.TOTAL_SUFFIX,
StreamMetricsRegistry.THREAD_LEVEL_GROUP)].Value);
Assert.AreEqual(0d,
closeTaskSensor.Metrics[MetricName.NameAndGroup(
ThreadMetrics.CLOSE_TASK + StreamMetricsRegistry.RATE_SUFFIX,
StreamMetricsRegistry.THREAD_LEVEL_GROUP)].Value);
var commitSensor = sensors.FirstOrDefault(s => s.Name.Equals(GetSensorName(ThreadMetrics.COMMIT)));
Assert.AreEqual(4, commitSensor.Metrics.Count());
Assert.IsTrue(
(double)commitSensor.Metrics[MetricName.NameAndGroup(
ThreadMetrics.COMMIT + StreamMetricsRegistry.TOTAL_SUFFIX,
StreamMetricsRegistry.THREAD_LEVEL_GROUP)].Value > 0d);
Assert.IsTrue(
(double)commitSensor.Metrics[MetricName.NameAndGroup(
ThreadMetrics.COMMIT + StreamMetricsRegistry.RATE_SUFFIX,
StreamMetricsRegistry.THREAD_LEVEL_GROUP)].Value > 0d);
Assert.IsTrue(
(double)commitSensor.Metrics[MetricName.NameAndGroup(
ThreadMetrics.COMMIT + StreamMetricsRegistry.LATENCY_SUFFIX + StreamMetricsRegistry.AVG_SUFFIX,
StreamMetricsRegistry.THREAD_LEVEL_GROUP)].Value > 0d);
Assert.IsTrue(
(double)commitSensor.Metrics[MetricName.NameAndGroup(
ThreadMetrics.COMMIT + StreamMetricsRegistry.LATENCY_SUFFIX + StreamMetricsRegistry.MAX_SUFFIX,
StreamMetricsRegistry.THREAD_LEVEL_GROUP)].Value > 0d);
var pollSensor = sensors.FirstOrDefault(s => s.Name.Equals(GetSensorName(ThreadMetrics.POLL)));
Assert.AreEqual(4, pollSensor.Metrics.Count());
Assert.IsTrue(
(double)pollSensor.Metrics[MetricName.NameAndGroup(
ThreadMetrics.POLL + StreamMetricsRegistry.TOTAL_SUFFIX,
StreamMetricsRegistry.THREAD_LEVEL_GROUP)].Value > 0d);
Assert.IsTrue(
(double)pollSensor.Metrics[MetricName.NameAndGroup(
ThreadMetrics.POLL + StreamMetricsRegistry.RATE_SUFFIX,
StreamMetricsRegistry.THREAD_LEVEL_GROUP)].Value > 0d);
Assert.IsTrue(
(double)pollSensor.Metrics[MetricName.NameAndGroup(
ThreadMetrics.POLL + StreamMetricsRegistry.LATENCY_SUFFIX + StreamMetricsRegistry.AVG_SUFFIX,
StreamMetricsRegistry.THREAD_LEVEL_GROUP)].Value > 0d);
Assert.IsTrue(
(double)pollSensor.Metrics[MetricName.NameAndGroup(
ThreadMetrics.POLL + StreamMetricsRegistry.LATENCY_SUFFIX + StreamMetricsRegistry.MAX_SUFFIX,
StreamMetricsRegistry.THREAD_LEVEL_GROUP)].Value > 0d);
var pollRecordsSensor = sensors.FirstOrDefault(s => s.Name.Equals(GetSensorName(ThreadMetrics.POLL + StreamMetricsRegistry.RECORDS_SUFFIX)));
Assert.AreEqual(2, pollRecordsSensor.Metrics.Count());
Assert.IsTrue(
(double)pollRecordsSensor.Metrics[MetricName.NameAndGroup(
ThreadMetrics.POLL + StreamMetricsRegistry.RECORDS_SUFFIX + StreamMetricsRegistry.AVG_SUFFIX,
StreamMetricsRegistry.THREAD_LEVEL_GROUP)].Value > 0d);
Assert.IsTrue(
(double)pollRecordsSensor.Metrics[MetricName.NameAndGroup(
ThreadMetrics.POLL + StreamMetricsRegistry.RECORDS_SUFFIX + StreamMetricsRegistry.MAX_SUFFIX,
StreamMetricsRegistry.THREAD_LEVEL_GROUP)].Value > 0d);
var processRecordsSensor = sensors.FirstOrDefault(s => s.Name.Equals(GetSensorName(ThreadMetrics.PROCESS + StreamMetricsRegistry.RECORDS_SUFFIX)));
Assert.AreEqual(2, processRecordsSensor.Metrics.Count());
Assert.IsTrue(
(double)processRecordsSensor.Metrics[MetricName.NameAndGroup(
ThreadMetrics.PROCESS + StreamMetricsRegistry.RECORDS_SUFFIX + StreamMetricsRegistry.AVG_SUFFIX,
StreamMetricsRegistry.THREAD_LEVEL_GROUP)].Value > 0d);
Assert.IsTrue(
(double)processRecordsSensor.Metrics[MetricName.NameAndGroup(
ThreadMetrics.PROCESS + StreamMetricsRegistry.RECORDS_SUFFIX + StreamMetricsRegistry.MAX_SUFFIX,
StreamMetricsRegistry.THREAD_LEVEL_GROUP)].Value > 0d);
var processRateSensor = sensors.FirstOrDefault(s => s.Name.Equals(GetSensorName(ThreadMetrics.PROCESS + StreamMetricsRegistry.RATE_SUFFIX)));
Assert.AreEqual(2, processRateSensor.Metrics.Count());
Assert.IsTrue(
(double)processRateSensor.Metrics[MetricName.NameAndGroup(
ThreadMetrics.PROCESS + StreamMetricsRegistry.RATE_SUFFIX + StreamMetricsRegistry.RATE_SUFFIX,
StreamMetricsRegistry.THREAD_LEVEL_GROUP)].Value > 0d);
Assert.IsTrue(
(double)processRateSensor.Metrics[MetricName.NameAndGroup(
ThreadMetrics.PROCESS + StreamMetricsRegistry.RATE_SUFFIX + StreamMetricsRegistry.TOTAL_SUFFIX,
StreamMetricsRegistry.THREAD_LEVEL_GROUP)].Value > 0d);
var processLatencySensor = sensors.FirstOrDefault(s => s.Name.Equals(GetSensorName(ThreadMetrics.PROCESS + StreamMetricsRegistry.LATENCY_SUFFIX)));
Assert.AreEqual(2, processLatencySensor.Metrics.Count());
Assert.IsTrue(
(double)processLatencySensor.Metrics[MetricName.NameAndGroup(
ThreadMetrics.PROCESS + StreamMetricsRegistry.LATENCY_SUFFIX + StreamMetricsRegistry.AVG_SUFFIX,
StreamMetricsRegistry.THREAD_LEVEL_GROUP)].Value > 0d);
Assert.IsTrue(
(double)processLatencySensor.Metrics[MetricName.NameAndGroup(
ThreadMetrics.PROCESS + StreamMetricsRegistry.LATENCY_SUFFIX + StreamMetricsRegistry.MAX_SUFFIX,
StreamMetricsRegistry.THREAD_LEVEL_GROUP)].Value > 0d);
// ratio sensors
var processRatioSensor = sensors.FirstOrDefault(s => s.Name.Equals(GetSensorName(ThreadMetrics.PROCESS + StreamMetricsRegistry.RATIO_SUFFIX)));
var pollRatioSensor = sensors.FirstOrDefault(s => s.Name.Equals(GetSensorName(ThreadMetrics.POLL + StreamMetricsRegistry.RATIO_SUFFIX)));
var commitRatioSensor = sensors.FirstOrDefault(s => s.Name.Equals(GetSensorName(ThreadMetrics.COMMIT + StreamMetricsRegistry.RATIO_SUFFIX)));
Assert.AreEqual(1, processRatioSensor.Metrics.Count());
Assert.AreEqual(1, pollRatioSensor.Metrics.Count());
Assert.AreEqual(1, commitRatioSensor.Metrics.Count());
var processRatioValue = (double)processRatioSensor.Metrics[MetricName.NameAndGroup(
ThreadMetrics.PROCESS + StreamMetricsRegistry.RATIO_SUFFIX,
StreamMetricsRegistry.THREAD_LEVEL_GROUP)].Value;
var pollRatioValue = (double)pollRatioSensor.Metrics[MetricName.NameAndGroup(
ThreadMetrics.POLL + StreamMetricsRegistry.RATIO_SUFFIX,
StreamMetricsRegistry.THREAD_LEVEL_GROUP)].Value;
var commitRatioValue = (double)commitRatioSensor.Metrics[MetricName.NameAndGroup(
ThreadMetrics.COMMIT + StreamMetricsRegistry.RATIO_SUFFIX,
StreamMetricsRegistry.THREAD_LEVEL_GROUP)].Value;
double total = Math.Round(processRatioValue + pollRatioValue + commitRatioValue, 2);
// we accept 10% of lost
Assert.IsTrue(total >= 0.90d);
}
private void ExportMetrics(long now)
{
var sensors = streamMetricsRegistry.GetThreadScopeSensor(threadId);
Task.Factory.StartNew(() => streamConfig.MetricsReporter?.Invoke(sensors));
}
public void TaskMetricsTest()
{
var serdes = new StringSerDes();
var cloneConfig = config.Clone();
cloneConfig.ApplicationId = "consume-test";
var producer = syncKafkaSupplier.GetProducer(cloneConfig.ToProducerConfig());
var consumer = syncKafkaSupplier.GetConsumer(cloneConfig.ToConsumerConfig("test-consum"), null);
consumer.Subscribe("topic2");
int nbMessage = 1000;
// produce 1000 messages to input topic
List <ConsumeResult <byte[], byte[]> > messages = new List <ConsumeResult <byte[], byte[]> >();
int offset = 0;
for (int i = 0; i < nbMessage; ++i)
{
messages.Add(
new ConsumeResult <byte[], byte[]>
{
Message = new Message <byte[], byte[]>
{
Key = serdes.Serialize($"key{i + 1}", new SerializationContext()),
Value = serdes.Serialize($"value{i + 1}", new SerializationContext())
},
TopicPartitionOffset = new TopicPartitionOffset(topicPartition, offset++)
});
}
task.AddRecords(messages);
while (task.CanProcess(DateTime.Now.GetMilliseconds()))
{
Assert.IsTrue(task.Process());
Assert.IsTrue(task.CommitNeeded);
task.Commit();
}
var messagesSink = new List <ConsumeResult <byte[], byte[]> >();
AssertExtensions.WaitUntil(() =>
{
messagesSink.AddRange(consumer.ConsumeRecords(TimeSpan.FromSeconds(1)));
return(messagesSink.Count < nbMessage);
}, TimeSpan.FromSeconds(5),
TimeSpan.FromMilliseconds(10));
long now = DateTime.Now.GetMilliseconds();
var sensors = streamMetricsRegistry.GetThreadScopeSensor(threadId);
var processorSensor = sensors.FirstOrDefault(s => s.Name.Equals(GetSensorName(TaskMetrics.PROCESS)));
Assert.AreEqual(2, processorSensor.Metrics.Count());
Assert.AreEqual(nbMessage,
processorSensor.Metrics[MetricName.NameAndGroup(
TaskMetrics.PROCESS + StreamMetricsRegistry.TOTAL_SUFFIX,
StreamMetricsRegistry.TASK_LEVEL_GROUP)].Value);
Assert.IsTrue(
(double)(processorSensor.Metrics[MetricName.NameAndGroup(
TaskMetrics.PROCESS + StreamMetricsRegistry.RATE_SUFFIX,
StreamMetricsRegistry.TASK_LEVEL_GROUP)].Value) > 0d);
var enforcedProcessorSensor = sensors.FirstOrDefault(s => s.Name.Equals(GetSensorName(TaskMetrics.ENFORCED_PROCESSING)));
Assert.AreEqual(2, enforcedProcessorSensor.Metrics.Count());
Assert.AreEqual(0,
enforcedProcessorSensor.Metrics[MetricName.NameAndGroup(
TaskMetrics.ENFORCED_PROCESSING + StreamMetricsRegistry.TOTAL_SUFFIX,
StreamMetricsRegistry.TASK_LEVEL_GROUP)].Value);
Assert.AreEqual(0,
enforcedProcessorSensor.Metrics[MetricName.NameAndGroup(
TaskMetrics.ENFORCED_PROCESSING + StreamMetricsRegistry.RATE_SUFFIX,
StreamMetricsRegistry.TASK_LEVEL_GROUP)].Value);
var processLatency = sensors.FirstOrDefault(s => s.Name.Equals(GetSensorName(TaskMetrics.PROCESS_LATENCY)));
Assert.AreEqual(2, processLatency.Metrics.Count());
Assert.IsTrue(
(double)processLatency.Metrics[MetricName.NameAndGroup(
TaskMetrics.PROCESS_LATENCY + StreamMetricsRegistry.AVG_SUFFIX,
StreamMetricsRegistry.TASK_LEVEL_GROUP)].Value > 0d);
Assert.IsTrue(
(double)processLatency.Metrics[MetricName.NameAndGroup(
TaskMetrics.PROCESS_LATENCY + StreamMetricsRegistry.MAX_SUFFIX,
StreamMetricsRegistry.TASK_LEVEL_GROUP)].Value > 0d);
var commitSensor = sensors.FirstOrDefault(s => s.Name.Equals(GetSensorName(TaskMetrics.COMMIT)));
Assert.AreEqual(2, commitSensor.Metrics.Count());
Assert.AreEqual(nbMessage,
commitSensor.Metrics[MetricName.NameAndGroup(
TaskMetrics.COMMIT + StreamMetricsRegistry.TOTAL_SUFFIX,
StreamMetricsRegistry.TASK_LEVEL_GROUP)].Value);
Assert.IsTrue(
(double)commitSensor.Metrics[MetricName.NameAndGroup(
TaskMetrics.COMMIT + StreamMetricsRegistry.RATE_SUFFIX,
StreamMetricsRegistry.TASK_LEVEL_GROUP)].Value > 0d);
var droppedRecordSensor = sensors.FirstOrDefault(s => s.Name.Equals(GetSensorName(TaskMetrics.DROPPED_RECORDS)));
Assert.AreEqual(2, droppedRecordSensor.Metrics.Count());
Assert.AreEqual(0,
droppedRecordSensor.Metrics[MetricName.NameAndGroup(
TaskMetrics.DROPPED_RECORDS + StreamMetricsRegistry.TOTAL_SUFFIX,
StreamMetricsRegistry.TASK_LEVEL_GROUP)].Value);
Assert.AreEqual(0,
droppedRecordSensor.Metrics[MetricName.NameAndGroup(
TaskMetrics.DROPPED_RECORDS + StreamMetricsRegistry.RATE_SUFFIX,
StreamMetricsRegistry.TASK_LEVEL_GROUP)].Value);
var activeBufferedRecordSensor = sensors.FirstOrDefault(s => s.Name.Equals(GetSensorName(TaskMetrics.ACTIVE_TASK_PREFIX + TaskMetrics.BUFFER_COUNT)));
Assert.AreEqual(1, activeBufferedRecordSensor.Metrics.Count());
Assert.AreEqual(0,
activeBufferedRecordSensor.Metrics[MetricName.NameAndGroup(
TaskMetrics.ACTIVE_TASK_PREFIX + TaskMetrics.BUFFER_COUNT,
StreamMetricsRegistry.TASK_LEVEL_GROUP)].Value);
var restorationRecordsSensor = sensors.FirstOrDefault(s => s.Name.Equals(GetSensorName(TaskMetrics.RESTORATION_RECORDS)));
Assert.AreEqual(1, restorationRecordsSensor.Metrics.Count());
Assert.AreEqual(0,
restorationRecordsSensor.Metrics[MetricName.NameAndGroup(
TaskMetrics.RESTORATION_RECORDS,
StreamMetricsRegistry.TASK_LEVEL_GROUP)].Value);
var activeRestorationSensor = sensors.FirstOrDefault(s => s.Name.Equals(GetSensorName(TaskMetrics.ACTIVE_RESTORATION)));
Assert.AreEqual(1, activeRestorationSensor.Metrics.Count());
Assert.AreEqual(0,
activeRestorationSensor.Metrics[MetricName.NameAndGroup(
TaskMetrics.ACTIVE_RESTORATION,
StreamMetricsRegistry.TASK_LEVEL_GROUP)].Value);
}
