public void AddMultipleRecordInMultiplePartitionDifferentDateTest() { var grouper = new PartitionGrouper(GetPartitions()); DateTime dt = DateTime.Now; grouper.AddRecord(topicPart1, MakeMessageWithKey("key", "1", dt.AddMilliseconds(1))); grouper.AddRecord(topicPart2, MakeMessageWithKey("key", "2", dt)); grouper.AddRecord(topicPart1, MakeMessageWithKey("key", "3", dt.AddMilliseconds(-13))); Assert.IsTrue(grouper.AllPartitionsBuffered); Assert.AreEqual(3, grouper.NumBuffered()); Assert.AreEqual(2, grouper.NumBuffered(topicPart1)); Assert.AreEqual(1, grouper.NumBuffered(topicPart2)); var record = grouper.NextRecord; Assert.IsNotNull(record); Assert.AreEqual("key", serdes.Deserialize(record.Record.Message.Key, new SerializationContext())); Assert.AreEqual("2", serdes.Deserialize(record.Record.Message.Value, new SerializationContext())); Assert.AreEqual("source2", record.Processor.Name); Assert.IsTrue(record.Queue.IsEmpty); record = grouper.NextRecord; Assert.IsNotNull(record); Assert.AreEqual("key", serdes.Deserialize(record.Record.Message.Key, new SerializationContext())); Assert.AreEqual("1", serdes.Deserialize(record.Record.Message.Value, new SerializationContext())); Assert.AreEqual("source1", record.Processor.Name); Assert.IsFalse(record.Queue.IsEmpty); record = grouper.NextRecord; Assert.IsNotNull(record); Assert.AreEqual("key", serdes.Deserialize(record.Record.Message.Key, new SerializationContext())); Assert.AreEqual("3", serdes.Deserialize(record.Record.Message.Value, new SerializationContext())); Assert.AreEqual("source1", record.Processor.Name); Assert.IsTrue(record.Queue.IsEmpty); Assert.IsNull(grouper.NextRecord); }
public void AddMultipleRecordInMultiplePartitionTest() { var grouper = new PartitionGrouper(GetPartitions()); grouper.AddRecord(topicPart1, MakeMessageWithKey("key", "test")); grouper.AddRecord(topicPart2, MakeMessageWithKey("key", "test2")); Assert.IsTrue(grouper.AllPartitionsBuffered); Assert.AreEqual(2, grouper.NumBuffered()); Assert.AreEqual(1, grouper.NumBuffered(topicPart1)); Assert.AreEqual(1, grouper.NumBuffered(topicPart2)); var record = grouper.NextRecord; Assert.IsNotNull(record); Assert.AreEqual("key", serdes.Deserialize(record.Record.Message.Key, new SerializationContext())); Assert.AreEqual("test", serdes.Deserialize(record.Record.Message.Value, new SerializationContext())); Assert.AreEqual("source1", record.Processor.Name); Assert.IsTrue(record.Queue.IsEmpty); record = grouper.NextRecord; Assert.IsNotNull(record); Assert.AreEqual("key", serdes.Deserialize(record.Record.Message.Key, new SerializationContext())); Assert.AreEqual("test2", serdes.Deserialize(record.Record.Message.Value, new SerializationContext())); Assert.AreEqual("source2", record.Processor.Name); Assert.IsTrue(record.Queue.IsEmpty); Assert.IsNull(grouper.NextRecord); }
public void AddOneRecordErrorTest() { var grouper = new PartitionGrouper(new Dictionary <TopicPartition, RecordQueue>()); Assert.Throws <IllegalStateException>(() => grouper.AddRecord(topicPart1, MakeMessageWithKey("key", "test"))); Assert.Throws <IllegalStateException>(() => grouper.NumBuffered(topicPart1)); }
public void AddOneRecordTest() { var grouper = new PartitionGrouper(GetPartitions()); grouper.AddRecord(topicPart1, MakeMessageWithKey("key", "test")); Assert.IsFalse(grouper.AllPartitionsBuffered); Assert.AreEqual(1, grouper.NumBuffered()); Assert.AreEqual(1, grouper.NumBuffered(topicPart1)); }
public StreamTask(string threadId, TaskId id, IEnumerable <TopicPartition> partitions, ProcessorTopology processorTopology, IConsumer <byte[], byte[]> consumer, IStreamConfig configuration, IKafkaSupplier kafkaSupplier, IProducer <byte[], byte[]> producer) : base(id, partitions, processorTopology, consumer, configuration) { this.threadId = threadId; this.kafkaSupplier = kafkaSupplier; consumedOffsets = new Dictionary <TopicPartition, long>(); maxTaskIdleMs = configuration.MaxTaskIdleMs; maxBufferedSize = configuration.BufferedRecordsPerPartition; followMetadata = configuration.FollowMetadata; idleStartTime = -1; // eos enabled if (producer == null) { this.producer = CreateEOSProducer(); InitializeTransaction(); eosEnabled = true; } else { this.producer = producer; } collector = new RecordCollector(logPrefix, configuration, id); collector.Init(ref this.producer); Context = new ProcessorContext(this, configuration, stateMgr).UseRecordCollector(collector); Context.FollowMetadata = followMetadata; var partitionsQueue = new Dictionary <TopicPartition, RecordQueue>(); foreach (var p in partitions) { var sourceProcessor = processorTopology.GetSourceProcessor(p.Topic); var sourceTimestampExtractor = sourceProcessor.Extractor ?? configuration.DefaultTimestampExtractor; var queue = new RecordQueue( logPrefix, $"record-queue-{p.Topic}-{id.Id}-{id.Partition}", sourceTimestampExtractor, p, sourceProcessor); partitionsQueue.Add(p, queue); processors.Add(sourceProcessor); } partitionGrouper = new PartitionGrouper(partitionsQueue); }
public StreamTask(string threadId, TaskId id, IEnumerable <TopicPartition> partitions, ProcessorTopology processorTopology, IConsumer <byte[], byte[]> consumer, IStreamConfig configuration, IKafkaSupplier kafkaSupplier, IProducer <byte[], byte[]> producer, IChangelogRegister changelogRegister, StreamMetricsRegistry streamMetricsRegistry) : base(id, partitions, processorTopology, consumer, configuration, changelogRegister) { this.threadId = threadId; this.kafkaSupplier = kafkaSupplier; this.streamMetricsRegistry = streamMetricsRegistry; consumedOffsets = new Dictionary <TopicPartition, long>(); maxTaskIdleMs = configuration.MaxTaskIdleMs; maxBufferedSize = configuration.BufferedRecordsPerPartition; followMetadata = configuration.FollowMetadata; idleStartTime = -1; // eos enabled if (producer == null) { this.producer = CreateEOSProducer(); InitializeTransaction(); eosEnabled = true; } else { this.producer = producer; } var droppedRecordsSensor = TaskMetrics.DroppedRecordsSensor(this.threadId, Id, this.streamMetricsRegistry); collector = new RecordCollector(logPrefix, configuration, id, droppedRecordsSensor); collector.Init(ref this.producer); Context = new ProcessorContext(this, configuration, stateMgr, streamMetricsRegistry) .UseRecordCollector(collector); Context.FollowMetadata = followMetadata; var partitionsQueue = new Dictionary <TopicPartition, RecordQueue>(); foreach (var p in partitions) { var sourceProcessor = processorTopology.GetSourceProcessor(p.Topic); sourceProcessor.SetTaskId(id); var sourceTimestampExtractor = sourceProcessor.Extractor ?? configuration.DefaultTimestampExtractor; var queue = new RecordQueue( logPrefix, $"record-queue-{p.Topic}-{id.Id}-{id.Partition}", sourceTimestampExtractor, p, sourceProcessor, droppedRecordsSensor); partitionsQueue.Add(p, queue); processors.Add(sourceProcessor); } partitionGrouper = new PartitionGrouper(partitionsQueue); closeTaskSensor = ThreadMetrics.ClosedTaskSensor(this.threadId, streamMetricsRegistry); activeBufferedRecordSensor = TaskMetrics.ActiveBufferedRecordsSensor(this.threadId, Id, streamMetricsRegistry); processSensor = TaskMetrics.ProcessSensor(this.threadId, Id, streamMetricsRegistry); processLatencySensor = TaskMetrics.ProcessLatencySensor(this.threadId, Id, streamMetricsRegistry); enforcedProcessingSensor = TaskMetrics.EnforcedProcessingSensor(this.threadId, Id, streamMetricsRegistry); commitSensor = TaskMetrics.CommitSensor(this.threadId, Id, streamMetricsRegistry); activeRestorationSensor = TaskMetrics.ActiveRestorationSensor(this.threadId, Id, streamMetricsRegistry); restorationRecordsSendsor = TaskMetrics.RestorationRecordsSensor(this.threadId, Id, streamMetricsRegistry); }