public async Task UpdateOrCreateOffsetConsumerGroupExistsTest()
{
// Creating a broker router and a protocol gateway for the producer and consumer
var brokerRouter = new BrokerRouter(_kafkaUri, new ConnectionFactory(), _config);
var partitionId = 0;
var consumerGroup = TestConfig.ConsumerName();
var topicName = TestConfig.TopicName();
var offest = 5;
var newOffset = 10;
await brokerRouter.GetTopicOffsetAsync(topicName, partitionId, CancellationToken.None);
await brokerRouter.CommitTopicOffsetAsync(topicName, partitionId, consumerGroup, offest, CancellationToken.None);
var res = await brokerRouter.GetTopicOffsetAsync(topicName, partitionId, consumerGroup, CancellationToken.None);
Assert.AreEqual(offest, res.Offset);
await brokerRouter.CommitTopicOffsetAsync(topicName, partitionId, consumerGroup, newOffset, CancellationToken.None);
res = await brokerRouter.GetTopicOffsetAsync(topicName, partitionId, consumerGroup, CancellationToken.None);
Assert.AreEqual(newOffset, res.Offset);
}
public async Task FetchMessagesCacheContainsAllRequestTest()
{
// Creating a broker router and a protocol gateway for the producer and consumer
var brokerRouter = new BrokerRouter(_options);
var producer = new Producer(brokerRouter);
var topic = TestConfig.TopicName();
var consumer = new Consumer(brokerRouter, _consumerConfig);
var offset = await brokerRouter.GetTopicOffsetAsync(topic, _partitionId, CancellationToken.None);
// Creating 5 messages
var messages = CreateTestMessages(10, 1);
await producer.SendMessagesAsync(messages, topic, _partitionId, new SendMessageConfiguration(ackTimeout : TimeSpan.FromSeconds(3)), CancellationToken.None);
// Now let's consume
var result = (await consumer.FetchMessagesAsync(offset, 5, CancellationToken.None)).ToList();
CheckMessages(messages.Take(5).ToList(), result);
// Now let's consume again
result = (await consumer.FetchMessagesAsync(offset.TopicName, offset.PartitionId, offset.Offset + 5, 5, CancellationToken.None)).ToList();
CheckMessages(messages.Skip(5).ToList(), result);
}
public async Task FetchMessagesBufferUnderRunTest()
{
// Creating a broker router and a protocol gateway for the producer and consumer
var brokerRouter = new BrokerRouter(_options);
var smallMessageSet = 4096 / 2;
var producer = new Producer(brokerRouter);
var topic = TestConfig.TopicName();
var consumer = new Consumer(brokerRouter, new ConsumerConfiguration(maxPartitionFetchBytes: smallMessageSet));
var offset = await brokerRouter.GetTopicOffsetAsync(topic, _partitionId, CancellationToken.None);
// Creating 5 messages
var messages = CreateTestMessages(10, 4096);
await producer.SendMessagesAsync(messages, topic, _partitionId, new SendMessageConfiguration(ackTimeout : TimeSpan.FromSeconds(3)), CancellationToken.None);
try {
// Now let's consume
await consumer.FetchMessagesAsync(offset, 5, CancellationToken.None);
Assert.Fail("should have thrown BufferUnderRunException");
} catch (BufferUnderRunException ex) {
Console.WriteLine(ex.ToString());
}
}
public async Task ProducerShouldUsePartitionIdInsteadOfMessageKeyToChoosePartition()
{
var partitionSelector = Substitute.For <IPartitionSelector>();
partitionSelector.Select(null, null)
.ReturnsForAnyArgs(_ => _.Arg <MetadataResponse.Topic>().Partitions.Single(p => p.PartitionId == 1));
using (var router = new BrokerRouter(new KafkaOptions(TestConfig.IntegrationUri, partitionSelector: partitionSelector))) {
var offset = await router.GetTopicOffsetAsync(TestConfig.TopicName(), 0, CancellationToken.None);
using (var producer = new Producer(router)) {
//message should send to PartitionId and not use the key to Select Broker Route !!
for (var i = 0; i < 20; i++)
{
await producer.SendMessageAsync(new Message(i.ToString(), "key"), offset.TopicName, offset.PartitionId, CancellationToken.None);
}
}
using (var consumer = new Consumer(router)) {
using (var source = new CancellationTokenSource()) {
var i = 0;
await consumer.FetchAsync(
offset, 20, message =>
{
Assert.That(message.Value.ToUtf8String(), Is.EqualTo(i++.ToString()));
if (i >= 20)
{
source.Cancel();
}
return(Task.FromResult(0));
}, source.Token);
}
}
}
}
public async Task FetchLastOffsetTopicDoesntExistTest()
{
// Creating a broker router and a protocol gateway for the producer and consumer
var brokerRouter = new BrokerRouter(_kafkaUri, new ConnectionFactory(), _config, log: TestConfig.InfoLog);
var topic = TestConfig.TopicName();
await brokerRouter.GetTopicOffsetAsync(topic, _partitionId, CancellationToken.None);
}
public async Task FetchLastOffsetPartitionDoesntExistTest()
{
// Creating a broker router and a protocol gateway for the producer and consumer
var brokerRouter = new BrokerRouter(_kafkaUri, new ConnectionFactory(), _config);
var partitionId = 100;
var topic = TestConfig.TopicName();
Assert.ThrowsAsync <CachedMetadataException>(async() => await brokerRouter.GetTopicOffsetAsync(topic, partitionId, CancellationToken.None));
}
public async Task FetchOffsetConsumerGroupDoesntExistTest()
{
// Creating a broker router and a protocol gateway for the producer and consumer
var brokerRouter = new BrokerRouter(_kafkaUri, new ConnectionFactory(), _config);
var partitionId = 0;
var consumerGroup = Guid.NewGuid().ToString();
var topicName = TestConfig.TopicName();
await brokerRouter.GetTopicOffsetAsync(topicName, partitionId, consumerGroup, CancellationToken.None);
}
public async Task FetchLastOffsetSimpleTest()
{
// Creating a broker router and a protocol gateway for the producer and consumer
var brokerRouter = new BrokerRouter(_kafkaUri, new ConnectionFactory(), _config);
var topic = TestConfig.TopicName();
var offset = await brokerRouter.GetTopicOffsetAsync(topic, _partitionId, CancellationToken.None);
Assert.AreNotEqual(-1, offset.Offset);
}
public async Task FetchMessagesNoNewMessagesInQueueTest()
{
// Creating a broker router and a protocol gateway for the producer and consumer
var brokerRouter = new BrokerRouter(_kafkaUri, new ConnectionFactory(), _config);
var consumer = new Consumer(brokerRouter, _consumerConfig);
var offset = await brokerRouter.GetTopicOffsetAsync(TestConfig.TopicName(), _partitionId, CancellationToken.None);
// Now let's consume
var result = (await consumer.FetchMessagesAsync(offset, 5, CancellationToken.None)).ToList();
Assert.AreEqual(0, result.Count, "Should not get any messages");
}
public async Task ManualConsumerFailure()
{
var topicName = "TestTopicIssue13-3R-1P";
using (var router = new BrokerRouter(_options)) {
var consumer = new Consumer(new BrokerRouter(_options), new ConsumerConfiguration(maxPartitionFetchBytes: 10000));
var offset = await router.GetTopicOffsetAsync(topicName, 0, CancellationToken.None);
var producer = new Producer(router);
var send = SandMessageForever(producer, offset.TopicName, offset.PartitionId);
var read = ReadMessageForever(consumer, offset.TopicName, offset.PartitionId, offset.Offset);
await Task.WhenAll(send, read);
}
}
public async Task FetchOffsetConsumerGroupArgumentNull([Values(null, "")] string group)
{
// Creating a broker router and a protocol gateway for the producer and consumer
var brokerRouter = new BrokerRouter(_kafkaUri, new ConnectionFactory(), _config);
var partitionId = 0;
var consumerGroup = TestConfig.ConsumerName();
var topicName = TestConfig.TopicName();
var offset = 5;
await brokerRouter.CommitTopicOffsetAsync(topicName, partitionId, consumerGroup, offset, CancellationToken.None);
Assert.ThrowsAsync <ArgumentNullException>(async() => await brokerRouter.GetTopicOffsetAsync(topicName, partitionId, group, CancellationToken.None));
}
public async Task FetchMessagesOffsetBiggerThanLastOffsetInQueueTest()
{
// Creating a broker router and a protocol gateway for the producer and consumer
var brokerRouter = new BrokerRouter(_kafkaUri, new ConnectionFactory(), _config, log: TestConfig.InfoLog);
var consumer = new Consumer(brokerRouter, _consumerConfig);
var offset = await brokerRouter.GetTopicOffsetAsync(TestConfig.TopicName(), _partitionId, CancellationToken.None);
try {
// Now let's consume
await consumer.FetchMessagesAsync(offset.TopicName, offset.PartitionId, offset.Offset + 1, 5, CancellationToken.None);
Assert.Fail("should have thrown FetchOutOfRangeException");
} catch (FetchOutOfRangeException ex) when(ex.Message.StartsWith("Kafka returned OffsetOutOfRange for Fetch request"))
{
Console.WriteLine(ex.ToString());
}
}
public async Task FetchMessagesSimpleTest()
{
// Creating a broker router and a protocol gateway for the producer and consumer
var brokerRouter = new BrokerRouter(_options);
var topic = "ManualConsumerTestTopic";
var producer = new Producer(brokerRouter);
var consumer = new Consumer(brokerRouter, _consumerConfig);
var offset = await brokerRouter.GetTopicOffsetAsync(topic, _partitionId, CancellationToken.None);
// Creating 5 messages
var messages = CreateTestMessages(5, 1);
await producer.SendMessagesAsync(messages, topic, _partitionId, new SendMessageConfiguration(ackTimeout : TimeSpan.FromSeconds(3)), CancellationToken.None);
// Now let's consume
var result = (await consumer.FetchMessagesAsync(offset, 5, CancellationToken.None)).ToList();
CheckMessages(messages, result);
}