public void AddMultipleRecordInMultiplePartitionDifferentDateTest()
{
var grouper = new PartitionGrouper(GetPartitions());
DateTime dt = DateTime.Now;
grouper.AddRecord(topicPart1, MakeMessageWithKey("key", "1", dt.AddMilliseconds(1)));
grouper.AddRecord(topicPart2, MakeMessageWithKey("key", "2", dt));
grouper.AddRecord(topicPart1, MakeMessageWithKey("key", "3", dt.AddMilliseconds(-13)));
Assert.IsTrue(grouper.AllPartitionsBuffered);
Assert.AreEqual(3, grouper.NumBuffered());
Assert.AreEqual(2, grouper.NumBuffered(topicPart1));
Assert.AreEqual(1, grouper.NumBuffered(topicPart2));
var record = grouper.NextRecord;
Assert.IsNotNull(record);
Assert.AreEqual("key", serdes.Deserialize(record.Record.Message.Key, new SerializationContext()));
Assert.AreEqual("2", serdes.Deserialize(record.Record.Message.Value, new SerializationContext()));
Assert.AreEqual("source2", record.Processor.Name);
Assert.IsTrue(record.Queue.IsEmpty);
record = grouper.NextRecord;
Assert.IsNotNull(record);
Assert.AreEqual("key", serdes.Deserialize(record.Record.Message.Key, new SerializationContext()));
Assert.AreEqual("1", serdes.Deserialize(record.Record.Message.Value, new SerializationContext()));
Assert.AreEqual("source1", record.Processor.Name);
Assert.IsFalse(record.Queue.IsEmpty);
record = grouper.NextRecord;
Assert.IsNotNull(record);
Assert.AreEqual("key", serdes.Deserialize(record.Record.Message.Key, new SerializationContext()));
Assert.AreEqual("3", serdes.Deserialize(record.Record.Message.Value, new SerializationContext()));
Assert.AreEqual("source1", record.Processor.Name);
Assert.IsTrue(record.Queue.IsEmpty);
Assert.IsNull(grouper.NextRecord);
}
public void AddMultipleRecordInMultiplePartitionTest()
{
var grouper = new PartitionGrouper(GetPartitions());
grouper.AddRecord(topicPart1, MakeMessageWithKey("key", "test"));
grouper.AddRecord(topicPart2, MakeMessageWithKey("key", "test2"));
Assert.IsTrue(grouper.AllPartitionsBuffered);
Assert.AreEqual(2, grouper.NumBuffered());
Assert.AreEqual(1, grouper.NumBuffered(topicPart1));
Assert.AreEqual(1, grouper.NumBuffered(topicPart2));
var record = grouper.NextRecord;
Assert.IsNotNull(record);
Assert.AreEqual("key", serdes.Deserialize(record.Record.Message.Key, new SerializationContext()));
Assert.AreEqual("test", serdes.Deserialize(record.Record.Message.Value, new SerializationContext()));
Assert.AreEqual("source1", record.Processor.Name);
Assert.IsTrue(record.Queue.IsEmpty);
record = grouper.NextRecord;
Assert.IsNotNull(record);
Assert.AreEqual("key", serdes.Deserialize(record.Record.Message.Key, new SerializationContext()));
Assert.AreEqual("test2", serdes.Deserialize(record.Record.Message.Value, new SerializationContext()));
Assert.AreEqual("source2", record.Processor.Name);
Assert.IsTrue(record.Queue.IsEmpty);
Assert.IsNull(grouper.NextRecord);
}
public void AddOneRecordErrorTest()
{
var grouper = new PartitionGrouper(new Dictionary <TopicPartition, RecordQueue>());
Assert.Throws <IllegalStateException>(() => grouper.AddRecord(topicPart1, MakeMessageWithKey("key", "test")));
Assert.Throws <IllegalStateException>(() => grouper.NumBuffered(topicPart1));
}
public void AddOneRecordTest()
{
var grouper = new PartitionGrouper(GetPartitions());
grouper.AddRecord(topicPart1, MakeMessageWithKey("key", "test"));
Assert.IsFalse(grouper.AllPartitionsBuffered);
Assert.AreEqual(1, grouper.NumBuffered());
Assert.AreEqual(1, grouper.NumBuffered(topicPart1));
}
public void AddRecord(ConsumeResult <byte[], byte[]> record)
{
int newQueueSize = partitionGrouper.AddRecord(record.TopicPartition, record);
if (newQueueSize > maxBufferedSize)
{
consumer.Pause(record.TopicPartition.ToSingle());
}
log.Debug($"{logPrefix}Added record into the buffered queue of partition {Partition}, new queue size is {newQueueSize}");
}