public void ConsumerShouldNotLoseMessageWhenBlocked()
{
var testId = Guid.NewGuid().ToString();
using (var router = new BrokerRouter(new KafkaOptions(TestConfig.IntegrationUri)))
using (var producer = new Producer(router))
{
var offsets = producer.BrokerRouter.GetTopicOffsetsAsync(TestConfig.TopicName(), CancellationToken.None).Result;
//create consumer with buffer size of 1 (should block upstream)
using (var consumer = new OldConsumer(new ConsumerOptions(TestConfig.TopicName(), router)
{
ConsumerBufferSize = 1, MaxWaitTimeForMinimumBytes = TimeSpan.Zero
},
offsets.Select(x => new OffsetPosition(x.PartitionId, x.Offset)).ToArray()))
{
for (var i = 0; i < 20; i++)
{
producer.SendMessageAsync(new Message(i.ToString(), testId), TestConfig.TopicName(), CancellationToken.None).Wait();
}
for (var i = 0; i < 20; i++)
{
var result = consumer.Consume().Take(1).First();
Assert.That(result.Key.ToUtf8String(), Is.EqualTo(testId));
Assert.That(result.Value.ToUtf8String(), Is.EqualTo(i.ToString()));
}
}
}
}
public async Task ConsumerShouldMoveToNextAvailableOffsetWhenQueryingForNextMessage()
{
const int expectedCount = 1000;
var options = new KafkaOptions(TestConfig.IntegrationUri);
using (var producerRouter = new BrokerRouter(options))
using (var producer = new Producer(producerRouter))
{
//get current offset and reset consumer to top of log
var offsets = await producer.BrokerRouter.GetTopicOffsetsAsync(TestConfig.TopicName(), CancellationToken.None).ConfigureAwait(false);
using (var consumerRouter = new BrokerRouter(options))
using (var consumer = new OldConsumer(new ConsumerOptions(TestConfig.TopicName(), consumerRouter)
{
MaxWaitTimeForMinimumBytes = TimeSpan.Zero
},
offsets.Select(x => new OffsetPosition(x.PartitionId, x.Offset)).ToArray()))
{
Console.WriteLine("Sending {0} test messages", expectedCount);
var response = await producer.SendMessagesAsync(Enumerable.Range(0, expectedCount).Select(x => new Message(x.ToString())), TestConfig.TopicName(), CancellationToken.None);
Assert.That(response.Any(x => x.ErrorCode != (int)ErrorResponseCode.None), Is.False, "Error occured sending test messages to server.");
var stream = consumer.Consume();
Console.WriteLine("Reading message back out from consumer.");
var data = stream.Take(expectedCount).ToList();
var consumerOffset = consumer.GetOffsetPosition().OrderBy(x => x.PartitionId).ToList();
var serverOffset = await producer.BrokerRouter.GetTopicOffsetsAsync(TestConfig.TopicName(), CancellationToken.None).ConfigureAwait(false);
var positionOffset = serverOffset.Select(x => new OffsetPosition(x.PartitionId, x.Offset))
.OrderBy(x => x.PartitionId)
.ToList();
Assert.That(consumerOffset, Is.EqualTo(positionOffset), "The consumerOffset position should match the server offset position.");
Assert.That(data.Count, Is.EqualTo(expectedCount), "We should have received 2000 messages from the server.");
}
}
}
