예제 #1
0
        public async Task FetchMessagesCacheContainsNoneOfRequestTest()
        {
            // Creating a broker router and a protocol gateway for the producer and consumer
            var brokerRouter    = new BrokerRouter(_options);
            var protocolGateway = new ProtocolGateway(_kafkaUri);

            Producer       producer = new Producer(brokerRouter);
            ManualConsumer consumer = new ManualConsumer(_partitionId, _topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize);

            var offset = await consumer.FetchLastOffset();

            // Creating 5 messages
            List <Message> messages = CreateTestMessages(10, 4096);

            await producer.SendMessageAsync(_topic, messages, partition : _partitionId, timeout : TimeSpan.FromSeconds(3));

            // Now let's consume
            var result = (await consumer.FetchMessages(7, offset)).ToList();

            CheckMessages(messages.Take(7).ToList(), result);

            // Now let's consume again
            result = (await consumer.FetchMessages(2, offset + 8)).ToList();

            CheckMessages(messages.Skip(8).ToList(), result);
        }
예제 #2
0
        private async Task ReadMessageForever(ManualConsumer manualConsumer, long offset)
        {
            while (true)
            {
                try
                {
                    var messages = await manualConsumer.FetchMessages(1000, offset);

                    if (messages.Any())
                    {
                        foreach (var message in messages)
                        {
                            _log.InfoFormat("Offset{0}  ", message.Meta.Offset);
                        }
                        offset = messages.Max(x => x.Meta.Offset) + 1;
                    }
                    else
                    {
                        await Task.Delay(100);
                    }
                }
                catch (Exception ex)
                {
                    _log.InfoFormat("can't read:\n" + ex);
                }
            }
        }
예제 #3
0
        public async Task FetchMessagesInvalidOffsetTest()
        {
            // Creating a broker router and a protocol gateway for the producer and consumer
            var protocolGateway = new ProtocolGateway(_kafkaUri);

            ManualConsumer consumer = new ManualConsumer(_partitionId, _topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize);

            var offset = -1;

            // Now let's consume
            await consumer.FetchMessages(5, offset);
        }
예제 #4
0
        public async Task FetchMessagesOffsetBiggerThanLastOffsetInQueueTest()
        {
            // Creating a broker router and a protocol gateway for the producer and consumer
            var protocolGateway = new ProtocolGateway(_kafkaUri);

            ManualConsumer consumer = new ManualConsumer(_partitionId, _topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize);

            var offset = await consumer.FetchLastOffset();

            // Now let's consume throw KafkaApplicationException
            await consumer.FetchMessages(5, offset + 1);
        }
예제 #5
0
        public async Task FetchMessagesPartitionDoesntExist()
        {
            // Creating a broker router and a protocol gateway for the producer and consumer
            var protocolGateway = new ProtocolGateway(_kafkaUri);
            var partitionId     = 100;
            var topic           = _topic;

            ManualConsumer consumer = new ManualConsumer(partitionId, topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize * 2);

            var offset = 0;

            await consumer.FetchMessages(5, offset);
        }
예제 #6
0
        public async Task FetchMessagesNoNewMessagesInQueueTest()
        {
            // Creating a broker router and a protocol gateway for the producer and consumer
            var protocolGateway = new ProtocolGateway(_kafkaUri);

            ManualConsumer consumer = new ManualConsumer(_partitionId, _topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize);

            var offset = await consumer.FetchLastOffset();

            // Now let's consume
            var result = (await consumer.FetchMessages(5, offset)).ToList();

            Assert.AreEqual(0, result.Count, "Should not get any messages");
        }
예제 #7
0
        public async Task FetchMessagesTopicDoesntExist()
        {
            // Creating a broker router and a protocol gateway for the producer and consumer
            var protocolGateway = new ProtocolGateway(_kafkaUri);

            var topic = _topic + Guid.NewGuid();

            ManualConsumer consumer = new ManualConsumer(_partitionId, topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize * 2);

            var offset = await consumer.FetchLastOffset();

            // Now let's consume
            var result = (await consumer.FetchMessages(5, offset)).ToList();

            Assert.AreEqual(0, result.Count);
        }
예제 #8
0
        public async Task FetchMessagesSimpleTest()
        {
            // Creating a broker router and a protocol gateway for the producer and consumer
            var brokerRouter    = new BrokerRouter(_options);
            var protocolGateway = new ProtocolGateway(_kafkaUri);

            var topic = "ManualConsumerTestTopic";

            Producer       producer = new Producer(brokerRouter);
            ManualConsumer consumer = new ManualConsumer(_partitionId, topic, protocolGateway, "TestClient", DefaultMaxMessageSetSize);

            var offset = await consumer.FetchLastOffset();

            // Creating 5 messages
            List <Message> messages = CreateTestMessages(5, 1);

            await producer.SendMessageAsync(topic, messages, partition : _partitionId, timeout : TimeSpan.FromSeconds(3));

            // Now let's consume
            var result = (await consumer.FetchMessages(5, offset)).ToList();

            CheckMessages(messages, result);
        }
예제 #9
0
        public async Task FetchMessagesBufferUnderRunTest()
        {
            // Creating a broker router and a protocol gateway for the producer and consumer
            var brokerRouter    = new BrokerRouter(_options);
            var protocolGateway = new ProtocolGateway(_kafkaUri);

            var smallMessageSet = 4096 / 2;

            Producer       producer = new Producer(brokerRouter);
            ManualConsumer consumer = new ManualConsumer(_partitionId, _topic, protocolGateway, "TestClient", smallMessageSet);

            var offset = await consumer.FetchLastOffset();

            // Creating 5 messages
            List <Message> messages = CreateTestMessages(10, 4096);

            await producer.SendMessageAsync(_topic, messages, partition : _partitionId, timeout : TimeSpan.FromSeconds(3));

            // Now let's consume
            var result = (await consumer.FetchMessages(5, offset)).ToList();

            CheckMessages(messages.Take(5).ToList(), result);
        }