public void AddMultipleRecordInMultiplePartitionDifferentDateTest()
{
var grouper = new PartitionGrouper(GetPartitions());
DateTime dt = DateTime.Now;
grouper.AddRecord(topicPart1, MakeMessageWithKey("key", "1", dt.AddMilliseconds(1)));
grouper.AddRecord(topicPart2, MakeMessageWithKey("key", "2", dt));
grouper.AddRecord(topicPart1, MakeMessageWithKey("key", "3", dt.AddMilliseconds(-13)));
Assert.IsTrue(grouper.AllPartitionsBuffered);
Assert.AreEqual(3, grouper.NumBuffered());
Assert.AreEqual(2, grouper.NumBuffered(topicPart1));
Assert.AreEqual(1, grouper.NumBuffered(topicPart2));
var record = grouper.NextRecord;
Assert.IsNotNull(record);
Assert.AreEqual("key", serdes.Deserialize(record.Record.Message.Key, new SerializationContext()));
Assert.AreEqual("2", serdes.Deserialize(record.Record.Message.Value, new SerializationContext()));
Assert.AreEqual("source2", record.Processor.Name);
Assert.IsTrue(record.Queue.IsEmpty);
record = grouper.NextRecord;
Assert.IsNotNull(record);
Assert.AreEqual("key", serdes.Deserialize(record.Record.Message.Key, new SerializationContext()));
Assert.AreEqual("1", serdes.Deserialize(record.Record.Message.Value, new SerializationContext()));
Assert.AreEqual("source1", record.Processor.Name);
Assert.IsFalse(record.Queue.IsEmpty);
record = grouper.NextRecord;
Assert.IsNotNull(record);
Assert.AreEqual("key", serdes.Deserialize(record.Record.Message.Key, new SerializationContext()));
Assert.AreEqual("3", serdes.Deserialize(record.Record.Message.Value, new SerializationContext()));
Assert.AreEqual("source1", record.Processor.Name);
Assert.IsTrue(record.Queue.IsEmpty);
Assert.IsNull(grouper.NextRecord);
}
public void AddMultipleRecordInMultiplePartitionTest()
{
var grouper = new PartitionGrouper(GetPartitions());
grouper.AddRecord(topicPart1, MakeMessageWithKey("key", "test"));
grouper.AddRecord(topicPart2, MakeMessageWithKey("key", "test2"));
Assert.IsTrue(grouper.AllPartitionsBuffered);
Assert.AreEqual(2, grouper.NumBuffered());
Assert.AreEqual(1, grouper.NumBuffered(topicPart1));
Assert.AreEqual(1, grouper.NumBuffered(topicPart2));
var record = grouper.NextRecord;
Assert.IsNotNull(record);
Assert.AreEqual("key", serdes.Deserialize(record.Record.Message.Key, new SerializationContext()));
Assert.AreEqual("test", serdes.Deserialize(record.Record.Message.Value, new SerializationContext()));
Assert.AreEqual("source1", record.Processor.Name);
Assert.IsTrue(record.Queue.IsEmpty);
record = grouper.NextRecord;
Assert.IsNotNull(record);
Assert.AreEqual("key", serdes.Deserialize(record.Record.Message.Key, new SerializationContext()));
Assert.AreEqual("test2", serdes.Deserialize(record.Record.Message.Value, new SerializationContext()));
Assert.AreEqual("source2", record.Processor.Name);
Assert.IsTrue(record.Queue.IsEmpty);
Assert.IsNull(grouper.NextRecord);
}
public void AddOneRecordErrorTest()
{
var grouper = new PartitionGrouper(new Dictionary <TopicPartition, RecordQueue>());
Assert.Throws <IllegalStateException>(() => grouper.AddRecord(topicPart1, MakeMessageWithKey("key", "test")));
Assert.Throws <IllegalStateException>(() => grouper.NumBuffered(topicPart1));
}
public void AddOneRecordTest()
{
var grouper = new PartitionGrouper(GetPartitions());
grouper.AddRecord(topicPart1, MakeMessageWithKey("key", "test"));
Assert.IsFalse(grouper.AllPartitionsBuffered);
Assert.AreEqual(1, grouper.NumBuffered());
Assert.AreEqual(1, grouper.NumBuffered(topicPart1));
}
public StreamTask(string threadId, TaskId id, IEnumerable <TopicPartition> partitions, ProcessorTopology processorTopology, IConsumer <byte[], byte[]> consumer, IStreamConfig configuration, IKafkaSupplier kafkaSupplier, IProducer <byte[], byte[]> producer)
: base(id, partitions, processorTopology, consumer, configuration)
{
this.threadId = threadId;
this.kafkaSupplier = kafkaSupplier;
consumedOffsets = new Dictionary <TopicPartition, long>();
maxTaskIdleMs = configuration.MaxTaskIdleMs;
maxBufferedSize = configuration.BufferedRecordsPerPartition;
followMetadata = configuration.FollowMetadata;
idleStartTime = -1;
// eos enabled
if (producer == null)
{
this.producer = CreateEOSProducer();
InitializeTransaction();
eosEnabled = true;
}
else
{
this.producer = producer;
}
collector = new RecordCollector(logPrefix, configuration, id);
collector.Init(ref this.producer);
Context = new ProcessorContext(this, configuration, stateMgr).UseRecordCollector(collector);
Context.FollowMetadata = followMetadata;
var partitionsQueue = new Dictionary <TopicPartition, RecordQueue>();
foreach (var p in partitions)
{
var sourceProcessor = processorTopology.GetSourceProcessor(p.Topic);
var sourceTimestampExtractor = sourceProcessor.Extractor ?? configuration.DefaultTimestampExtractor;
var queue = new RecordQueue(
logPrefix,
$"record-queue-{p.Topic}-{id.Id}-{id.Partition}",
sourceTimestampExtractor,
p,
sourceProcessor);
partitionsQueue.Add(p, queue);
processors.Add(sourceProcessor);
}
partitionGrouper = new PartitionGrouper(partitionsQueue);
}
public StreamTask(string threadId, TaskId id, IEnumerable <TopicPartition> partitions,
ProcessorTopology processorTopology, IConsumer <byte[], byte[]> consumer, IStreamConfig configuration,
IKafkaSupplier kafkaSupplier, IProducer <byte[], byte[]> producer, IChangelogRegister changelogRegister,
StreamMetricsRegistry streamMetricsRegistry)
: base(id, partitions, processorTopology, consumer, configuration, changelogRegister)
{
this.threadId = threadId;
this.kafkaSupplier = kafkaSupplier;
this.streamMetricsRegistry = streamMetricsRegistry;
consumedOffsets = new Dictionary <TopicPartition, long>();
maxTaskIdleMs = configuration.MaxTaskIdleMs;
maxBufferedSize = configuration.BufferedRecordsPerPartition;
followMetadata = configuration.FollowMetadata;
idleStartTime = -1;
// eos enabled
if (producer == null)
{
this.producer = CreateEOSProducer();
InitializeTransaction();
eosEnabled = true;
}
else
{
this.producer = producer;
}
var droppedRecordsSensor = TaskMetrics.DroppedRecordsSensor(this.threadId, Id, this.streamMetricsRegistry);
collector = new RecordCollector(logPrefix, configuration, id, droppedRecordsSensor);
collector.Init(ref this.producer);
Context = new ProcessorContext(this, configuration, stateMgr, streamMetricsRegistry)
.UseRecordCollector(collector);
Context.FollowMetadata = followMetadata;
var partitionsQueue = new Dictionary <TopicPartition, RecordQueue>();
foreach (var p in partitions)
{
var sourceProcessor = processorTopology.GetSourceProcessor(p.Topic);
sourceProcessor.SetTaskId(id);
var sourceTimestampExtractor = sourceProcessor.Extractor ?? configuration.DefaultTimestampExtractor;
var queue = new RecordQueue(
logPrefix,
$"record-queue-{p.Topic}-{id.Id}-{id.Partition}",
sourceTimestampExtractor,
p,
sourceProcessor,
droppedRecordsSensor);
partitionsQueue.Add(p, queue);
processors.Add(sourceProcessor);
}
partitionGrouper = new PartitionGrouper(partitionsQueue);
closeTaskSensor = ThreadMetrics.ClosedTaskSensor(this.threadId, streamMetricsRegistry);
activeBufferedRecordSensor = TaskMetrics.ActiveBufferedRecordsSensor(this.threadId, Id, streamMetricsRegistry);
processSensor = TaskMetrics.ProcessSensor(this.threadId, Id, streamMetricsRegistry);
processLatencySensor = TaskMetrics.ProcessLatencySensor(this.threadId, Id, streamMetricsRegistry);
enforcedProcessingSensor = TaskMetrics.EnforcedProcessingSensor(this.threadId, Id, streamMetricsRegistry);
commitSensor = TaskMetrics.CommitSensor(this.threadId, Id, streamMetricsRegistry);
activeRestorationSensor = TaskMetrics.ActiveRestorationSensor(this.threadId, Id, streamMetricsRegistry);
restorationRecordsSendsor = TaskMetrics.RestorationRecordsSensor(this.threadId, Id, streamMetricsRegistry);
}
