public async Task UpdateOrCreateOffsetConsumerGroupExistsTest()
        {
            // Creating a broker router and a protocol gateway for the producer and consumer
            var brokerRouter  = new BrokerRouter(_kafkaUri, new ConnectionFactory(), _config);
            var partitionId   = 0;
            var consumerGroup = TestConfig.ConsumerName();

            var topicName = TestConfig.TopicName();
            var offest    = 5;
            var newOffset = 10;

            await brokerRouter.GetTopicOffsetAsync(topicName, partitionId, CancellationToken.None);

            await brokerRouter.CommitTopicOffsetAsync(topicName, partitionId, consumerGroup, offest, CancellationToken.None);

            var res = await brokerRouter.GetTopicOffsetAsync(topicName, partitionId, consumerGroup, CancellationToken.None);

            Assert.AreEqual(offest, res.Offset);

            await brokerRouter.CommitTopicOffsetAsync(topicName, partitionId, consumerGroup, newOffset, CancellationToken.None);

            res = await brokerRouter.GetTopicOffsetAsync(topicName, partitionId, consumerGroup, CancellationToken.None);

            Assert.AreEqual(newOffset, res.Offset);
        }
        public async Task FetchMessagesCacheContainsAllRequestTest()
        {
            // Creating a broker router and a protocol gateway for the producer and consumer
            var brokerRouter = new BrokerRouter(_options);

            var producer = new Producer(brokerRouter);
            var topic    = TestConfig.TopicName();
            var consumer = new Consumer(brokerRouter, _consumerConfig);

            var offset = await brokerRouter.GetTopicOffsetAsync(topic, _partitionId, CancellationToken.None);

            // Creating 5 messages
            var messages = CreateTestMessages(10, 1);

            await producer.SendMessagesAsync(messages, topic, _partitionId, new SendMessageConfiguration(ackTimeout : TimeSpan.FromSeconds(3)), CancellationToken.None);

            // Now let's consume
            var result = (await consumer.FetchMessagesAsync(offset, 5, CancellationToken.None)).ToList();

            CheckMessages(messages.Take(5).ToList(), result);

            // Now let's consume again
            result = (await consumer.FetchMessagesAsync(offset.TopicName, offset.PartitionId, offset.Offset + 5, 5, CancellationToken.None)).ToList();

            CheckMessages(messages.Skip(5).ToList(), result);
        }
        public async Task FetchMessagesBufferUnderRunTest()
        {
            // Creating a broker router and a protocol gateway for the producer and consumer
            var brokerRouter = new BrokerRouter(_options);

            var smallMessageSet = 4096 / 2;

            var producer = new Producer(brokerRouter);
            var topic    = TestConfig.TopicName();
            var consumer = new Consumer(brokerRouter, new ConsumerConfiguration(maxPartitionFetchBytes: smallMessageSet));

            var offset = await brokerRouter.GetTopicOffsetAsync(topic, _partitionId, CancellationToken.None);

            // Creating 5 messages
            var messages = CreateTestMessages(10, 4096);

            await producer.SendMessagesAsync(messages, topic, _partitionId, new SendMessageConfiguration(ackTimeout : TimeSpan.FromSeconds(3)), CancellationToken.None);

            try {
                // Now let's consume
                await consumer.FetchMessagesAsync(offset, 5, CancellationToken.None);

                Assert.Fail("should have thrown BufferUnderRunException");
            } catch (BufferUnderRunException ex) {
                Console.WriteLine(ex.ToString());
            }
        }
        public async Task ProducerShouldUsePartitionIdInsteadOfMessageKeyToChoosePartition()
        {
            var partitionSelector = Substitute.For <IPartitionSelector>();

            partitionSelector.Select(null, null)
            .ReturnsForAnyArgs(_ => _.Arg <MetadataResponse.Topic>().Partitions.Single(p => p.PartitionId == 1));

            using (var router = new BrokerRouter(new KafkaOptions(TestConfig.IntegrationUri, partitionSelector: partitionSelector))) {
                var offset = await router.GetTopicOffsetAsync(TestConfig.TopicName(), 0, CancellationToken.None);

                using (var producer = new Producer(router)) {
                    //message should send to PartitionId and not use the key to Select Broker Route !!
                    for (var i = 0; i < 20; i++)
                    {
                        await producer.SendMessageAsync(new Message(i.ToString(), "key"), offset.TopicName, offset.PartitionId, CancellationToken.None);
                    }
                }

                using (var consumer = new Consumer(router)) {
                    using (var source = new CancellationTokenSource()) {
                        var i = 0;
                        await consumer.FetchAsync(
                            offset, 20, message =>
                        {
                            Assert.That(message.Value.ToUtf8String(), Is.EqualTo(i++.ToString()));
                            if (i >= 20)
                            {
                                source.Cancel();
                            }
                            return(Task.FromResult(0));
                        }, source.Token);
                    }
                }
            }
        }
        public async Task FetchLastOffsetTopicDoesntExistTest()
        {
            // Creating a broker router and a protocol gateway for the producer and consumer
            var brokerRouter = new BrokerRouter(_kafkaUri, new ConnectionFactory(), _config, log: TestConfig.InfoLog);

            var topic = TestConfig.TopicName();

            await brokerRouter.GetTopicOffsetAsync(topic, _partitionId, CancellationToken.None);
        }
        public async Task FetchLastOffsetPartitionDoesntExistTest()
        {
            // Creating a broker router and a protocol gateway for the producer and consumer
            var brokerRouter = new BrokerRouter(_kafkaUri, new ConnectionFactory(), _config);
            var partitionId  = 100;
            var topic        = TestConfig.TopicName();

            Assert.ThrowsAsync <CachedMetadataException>(async() => await brokerRouter.GetTopicOffsetAsync(topic, partitionId, CancellationToken.None));
        }
        public async Task FetchOffsetConsumerGroupDoesntExistTest()
        {
            // Creating a broker router and a protocol gateway for the producer and consumer
            var brokerRouter  = new BrokerRouter(_kafkaUri, new ConnectionFactory(), _config);
            var partitionId   = 0;
            var consumerGroup = Guid.NewGuid().ToString();

            var topicName = TestConfig.TopicName();
            await brokerRouter.GetTopicOffsetAsync(topicName, partitionId, consumerGroup, CancellationToken.None);
        }
        public async Task FetchLastOffsetSimpleTest()
        {
            // Creating a broker router and a protocol gateway for the producer and consumer
            var brokerRouter = new BrokerRouter(_kafkaUri, new ConnectionFactory(), _config);

            var topic = TestConfig.TopicName();

            var offset = await brokerRouter.GetTopicOffsetAsync(topic, _partitionId, CancellationToken.None);

            Assert.AreNotEqual(-1, offset.Offset);
        }
        public async Task FetchMessagesNoNewMessagesInQueueTest()
        {
            // Creating a broker router and a protocol gateway for the producer and consumer
            var brokerRouter = new BrokerRouter(_kafkaUri, new ConnectionFactory(), _config);

            var consumer = new Consumer(brokerRouter, _consumerConfig);

            var offset = await brokerRouter.GetTopicOffsetAsync(TestConfig.TopicName(), _partitionId, CancellationToken.None);

            // Now let's consume
            var result = (await consumer.FetchMessagesAsync(offset, 5, CancellationToken.None)).ToList();

            Assert.AreEqual(0, result.Count, "Should not get any messages");
        }
Beispiel #10
0
        public async Task ManualConsumerFailure()
        {
            var topicName = "TestTopicIssue13-3R-1P";

            using (var router = new BrokerRouter(_options)) {
                var consumer = new Consumer(new BrokerRouter(_options), new ConsumerConfiguration(maxPartitionFetchBytes: 10000));
                var offset   = await router.GetTopicOffsetAsync(topicName, 0, CancellationToken.None);

                var producer = new Producer(router);
                var send     = SandMessageForever(producer, offset.TopicName, offset.PartitionId);
                var read     = ReadMessageForever(consumer, offset.TopicName, offset.PartitionId, offset.Offset);
                await Task.WhenAll(send, read);
            }
        }
        public async Task FetchOffsetConsumerGroupArgumentNull([Values(null, "")] string group)
        {
            // Creating a broker router and a protocol gateway for the producer and consumer
            var brokerRouter  = new BrokerRouter(_kafkaUri, new ConnectionFactory(), _config);
            var partitionId   = 0;
            var consumerGroup = TestConfig.ConsumerName();

            var topicName = TestConfig.TopicName();

            var offset = 5;

            await brokerRouter.CommitTopicOffsetAsync(topicName, partitionId, consumerGroup, offset, CancellationToken.None);

            Assert.ThrowsAsync <ArgumentNullException>(async() => await brokerRouter.GetTopicOffsetAsync(topicName, partitionId, group, CancellationToken.None));
        }
        public async Task FetchMessagesOffsetBiggerThanLastOffsetInQueueTest()
        {
            // Creating a broker router and a protocol gateway for the producer and consumer
            var brokerRouter = new BrokerRouter(_kafkaUri, new ConnectionFactory(), _config, log: TestConfig.InfoLog);

            var consumer = new Consumer(brokerRouter, _consumerConfig);

            var offset = await brokerRouter.GetTopicOffsetAsync(TestConfig.TopicName(), _partitionId, CancellationToken.None);

            try {
                // Now let's consume
                await consumer.FetchMessagesAsync(offset.TopicName, offset.PartitionId, offset.Offset + 1, 5, CancellationToken.None);

                Assert.Fail("should have thrown FetchOutOfRangeException");
            } catch (FetchOutOfRangeException ex) when(ex.Message.StartsWith("Kafka returned OffsetOutOfRange for Fetch request"))
            {
                Console.WriteLine(ex.ToString());
            }
        }
        public async Task FetchMessagesSimpleTest()
        {
            // Creating a broker router and a protocol gateway for the producer and consumer
            var brokerRouter = new BrokerRouter(_options);

            var topic = "ManualConsumerTestTopic";

            var producer = new Producer(brokerRouter);
            var consumer = new Consumer(brokerRouter, _consumerConfig);

            var offset = await brokerRouter.GetTopicOffsetAsync(topic, _partitionId, CancellationToken.None);

            // Creating 5 messages
            var messages = CreateTestMessages(5, 1);

            await producer.SendMessagesAsync(messages, topic, _partitionId, new SendMessageConfiguration(ackTimeout : TimeSpan.FromSeconds(3)), CancellationToken.None);

            // Now let's consume
            var result = (await consumer.FetchMessagesAsync(offset, 5, CancellationToken.None)).ToList();

            CheckMessages(messages, result);
        }