public void ConsumerShouldBeAbleToGetCurrentOffsetInformation() { using (var router = new BrokerRouter(new KafkaOptions(IntegrationConfig.IntegrationUri))) using (var producer = new Producer(router)) { var startOffsets = producer.GetTopicOffsetAsync(IntegrationConfig.IntegrationTopic).Result .Select(x => new OffsetPosition(x.PartitionId, x.Offsets.Max())).ToArray(); using (var consumer = new BlockingConsumer(new ConsumerOptions(IntegrationConfig.IntegrationTopic, router), startOffsets)) { for (int i = 0; i < 20; i++) { producer.SendMessageAsync(IntegrationConfig.IntegrationTopic, new[] { new Message(i.ToString(), "1") }).Wait(); } var results = consumer.Consume().Take(20).ToList(); //ensure the produced messages arrived for (int i = 0; i < 20; i++) { Assert.That(results[i].Value.ToUtf8String(), Is.EqualTo(i.ToString())); } //the current offsets should be 20 positions higher than start var currentOffsets = consumer.GetOffsetPosition(); Assert.That(currentOffsets.Sum(x => x.Offset) - startOffsets.Sum(x => x.Offset), Is.EqualTo(20)); } } }
public async void ConsumerShouldMoveToNextAvailableOffsetWhenQueryingForNextMessage() { const int expectedCount = 1000; var options = new KafkaOptions(IntegrationConfig.IntegrationUri) { Log = new ConsoleLog() }; using (var producerRouter = new BrokerRouter(options)) using (var producer = new Producer(producerRouter)) { //get current offset and reset consumer to top of log var offsets = await producer.GetTopicOffsetAsync(IntegrationConfig.IntegrationTopic).ConfigureAwait(false); using (var consumerRouter = new BrokerRouter(options)) using (var consumer = new BlockingConsumer(new ConsumerOptions(IntegrationConfig.IntegrationTopic, consumerRouter), offsets.Select(x => new OffsetPosition(x.PartitionId, x.Offsets.Max())).ToArray())) { Console.WriteLine("Sending {0} test messages", expectedCount); var response = await producer.SendMessageAsync(IntegrationConfig.IntegrationTopic, Enumerable.Range(0, expectedCount).Select(x => new Message(x.ToString()))); Assert.That(response.Any(x => x.Error != (int)ErrorResponseCode.NoError), Is.False, "Error occured sending test messages to server."); var stream = consumer.Consume(); Console.WriteLine("Reading message back out from consumer."); var data = stream.Take(expectedCount).ToList(); var consumerOffset = consumer.GetOffsetPosition().OrderBy(x => x.Offset).ToList(); var serverOffset = await producer.GetTopicOffsetAsync(IntegrationConfig.IntegrationTopic).ConfigureAwait(false); var positionOffset = serverOffset.Select(x => new OffsetPosition(x.PartitionId, x.Offsets.Max())) .OrderBy(x => x.Offset) .ToList(); Assert.That(consumerOffset, Is.EqualTo(positionOffset), "The consumerOffset position should match the server offset position."); Assert.That(data.Count, Is.EqualTo(expectedCount), "We should have received 2000 messages from the server."); } } }