public void ShouldThrowMessageTooLarge()
 {
     var connection = new Mock<IKafkaConnection>();
     var config = new SyncProducerConfiguration { MaxMessageSize = 99 };
     var producer = new SyncProducer(config, connection.Object);
     producer.Send(new ProducerRequest(1, "client", 0, 0, new List<TopicData>()
     {
         new TopicData("test",
             new List<PartitionData>()
             {
                 new PartitionData(0, new BufferedMessageSet(new List<Message>() {new Message(new byte[100])}, 0))
             })
     }));
 }
Exemple #2
0
        public void SimpleSyncProducerSendsLotsOfMessagesAndConsumerConnectorGetsThemBackWithVerySmallAutoCommitInterval()
        {
            var prodConfig     = this.SyncProducerConfig1;
            var consumerConfig = this.ZooKeeperBasedConsumerConfig;

            consumerConfig.AutoCommit         = true;
            consumerConfig.AutoCommitInterval = 10;
            int numberOfMessages = 500;
            int messageSize      = 0;

            List <Message> messagesToSend = new List <Message>();

            using (var producer = new SyncProducer(prodConfig))
            {
                for (int i = 0; i < numberOfMessages; i++)
                {
                    string payload1     = "kafka 1.";
                    byte[] payloadData1 = Encoding.UTF8.GetBytes(payload1);
                    var    msg          = new Message(payloadData1);
                    messagesToSend.Add(msg);
                    if (i == 0)
                    {
                        messageSize = msg.Size;
                    }
                    producer.Send(CurrentTestTopic, 0, new List <Message>()
                    {
                        msg
                    });
                }
            }

            Thread.Sleep(2000);

            // now consuming
            int resultCount = 0;

            using (IConsumerConnector consumerConnector = new ZookeeperConsumerConnector(consumerConfig, true))
            {
                var topicCount = new Dictionary <string, int> {
                    { CurrentTestTopic, 1 }
                };
                var messages = consumerConnector.CreateMessageStreams(topicCount);
                var sets     = messages[CurrentTestTopic];

                try
                {
                    foreach (var set in sets)
                    {
                        foreach (var message in set)
                        {
                            Assert.AreEqual(messageSize, message.Size);
                            resultCount++;
                        }
                    }
                }
                catch (ConsumerTimeoutException)
                {
                    // do nothing, this is expected
                }
            }

            Assert.AreEqual(numberOfMessages, resultCount);
        }
Exemple #3
0
 public KafkaProducerException(SyncProducer syncProducer)
 {
     this.innerProducer = syncProducer;
 }
Exemple #4
0
        public void SimpleSyncProducerSendsLotsOfTwiceCompressedMessagesAndConsumerConnectorGetsThemBack()
        {
            var prodConfig              = this.SyncProducerConfig1;
            var consumerConfig          = this.ZooKeeperBasedConsumerConfig;
            int numberOfMessages        = 500;
            int messagesPerPackage      = 5;
            int messageSize             = 0;
            int messagesPerInnerPackage = 5;

            using (var producer = new SyncProducer(prodConfig))
            {
                for (int i = 0; i < numberOfMessages; i++)
                {
                    var messagePackageList = new List <Message>();
                    for (int messageInPackageNr = 0; messageInPackageNr < messagesPerPackage; messageInPackageNr++)
                    {
                        var innerMessagePackageList = new List <Message>();
                        for (int inner = 0; inner < messagesPerInnerPackage; inner++)
                        {
                            string payload1     = "kafka 1.";
                            byte[] payloadData1 = Encoding.UTF8.GetBytes(payload1);
                            var    msg          = new Message(payloadData1);
                            innerMessagePackageList.Add(msg);
                        }
                        var innerPackageMessage = CompressionUtils.Compress(innerMessagePackageList, CompressionCodecs.GZIPCompressionCodec);
                        messagePackageList.Add(innerPackageMessage);
                    }
                    var packageMessage = CompressionUtils.Compress(messagePackageList, CompressionCodecs.GZIPCompressionCodec);

                    producer.Send(CurrentTestTopic, 0, new List <Message>()
                    {
                        packageMessage
                    });
                }
            }

            Thread.Sleep(2000);

            // now consuming
            int resultCount = 0;

            using (IConsumerConnector consumerConnector = new ZookeeperConsumerConnector(consumerConfig, true))
            {
                var topicCount = new Dictionary <string, int> {
                    { CurrentTestTopic, 1 }
                };
                var messages = consumerConnector.CreateMessageStreams(topicCount);
                var sets     = messages[CurrentTestTopic];

                try
                {
                    foreach (var set in sets)
                    {
                        foreach (var message in set)
                        {
                            resultCount++;
                        }
                    }
                }
                catch (ConsumerTimeoutException)
                {
                    // do nothing, this is expected
                }
            }

            Assert.AreEqual(numberOfMessages * messagesPerPackage * messagesPerInnerPackage, resultCount);
        }
Exemple #5
0
        public void SimpleSyncProducerSendsLotsOfCompressedMessagesWithIncreasedSizeAndConsumerConnectorGetsThemBack()
        {
            var prodConfig     = this.SyncProducerConfig1;
            var consumerConfig = this.ZooKeeperBasedConsumerConfig;

            consumerConfig.AutoCommit         = true;
            consumerConfig.AutoCommitInterval = 100;
            int    numberOfMessages   = 2000;
            int    messagesPerPackage = 5;
            string topic = CurrentTestTopic;

            var multipleBrokersHelper = new TestMultipleBrokersHelper(CurrentTestTopic);

            multipleBrokersHelper.GetCurrentOffsets(
                new[] { prodConfig });

            int  msgNr     = 0;
            long totalSize = 0;

            using (var producer = new SyncProducer(prodConfig))
            {
                for (int i = 0; i < numberOfMessages; i++)
                {
                    var messagePackageList = new List <Message>();
                    for (int messageInPackageNr = 0; messageInPackageNr < messagesPerPackage; messageInPackageNr++)
                    {
                        string payload1     = CreatePayloadByNumber(msgNr);
                        byte[] payloadData1 = Encoding.UTF8.GetBytes(payload1);
                        var    msg          = new Message(payloadData1);
                        totalSize += msg.Size;
                        messagePackageList.Add(msg);
                        msgNr++;
                    }
                    var packageMessage = CompressionUtils.Compress(messagePackageList, CompressionCodecs.GZIPCompressionCodec);
                    producer.Send(topic, 0, new List <Message>()
                    {
                        packageMessage
                    });
                }
            }

            // now consuming
            int  resultCount = 0;
            long resultSize  = 0;

            using (IConsumerConnector consumerConnector = new ZookeeperConsumerConnector(consumerConfig, true))
            {
                var topicCount = new Dictionary <string, int> {
                    { topic, 1 }
                };
                var messages = consumerConnector.CreateMessageStreams(topicCount);
                var sets     = messages[topic];

                try
                {
                    foreach (var set in sets)
                    {
                        foreach (var message in set)
                        {
                            Assert.AreEqual(CreatePayloadByNumber(resultCount), Encoding.UTF8.GetString(message.Payload));
                            resultCount++;
                            resultSize += message.Size;
                        }
                    }
                }
                catch (ConsumerTimeoutException)
                {
                    // do nothing, this is expected
                }
            }

            Assert.AreEqual(numberOfMessages * messagesPerPackage, resultCount);
            Assert.AreEqual(totalSize, resultSize);
        }
Exemple #6
0
        public void SimpleSyncProducerSends2MessagesAndConsumerConnectorGetsThemBack()
        {
            var prodConfig     = this.SyncProducerConfig1;
            var consumerConfig = this.ZooKeeperBasedConsumerConfig;
            var consConf       = this.ConsumerConfig1;

            // first producing
            string payload1 = "kafka 1.";

            byte[] payloadData1 = Encoding.UTF8.GetBytes(payload1);
            var    msg1         = new Message(payloadData1);

            string payload2 = "kafka 2.";

            byte[] payloadData2 = Encoding.UTF8.GetBytes(payload2);
            var    msg2         = new Message(payloadData2);

            var producerRequest = new ProducerRequest(CurrentTestTopic, 0, new List <Message> {
                msg1, msg2
            });

            using (var producer = new SyncProducer(prodConfig))
            {
                producer.Send(producerRequest);
            }

            var  consumer = new Consumer(consConf);
            long offset   = 0;
            var  result   = consumer.Fetch(
                new FetchRequest(CurrentTestTopic, 0, offset, 400));

            foreach (var resultItem in result)
            {
                offset += resultItem.Offset;
            }

            // now consuming
            var resultMessages = new List <Message>();

            using (IConsumerConnector consumerConnector = new ZookeeperConsumerConnector(consumerConfig, true))
            {
                var topicCount = new Dictionary <string, int> {
                    { CurrentTestTopic, 1 }
                };
                var messages = consumerConnector.CreateMessageStreams(topicCount);
                var sets     = messages[CurrentTestTopic];
                try
                {
                    foreach (var set in sets)
                    {
                        foreach (var message in set)
                        {
                            resultMessages.Add(message);
                        }
                    }
                }
                catch (ConsumerTimeoutException)
                {
                    // do nothing, this is expected
                }
            }

            Assert.AreEqual(2, resultMessages.Count);
            Assert.AreEqual(msg1.ToString(), resultMessages[0].ToString());
            Assert.AreEqual(msg2.ToString(), resultMessages[1].ToString());
        }
Exemple #7
0
        public void ConsumerConnectorConsumesTwoDifferentTopics()
        {
            var prodConfig     = this.SyncProducerConfig1;
            var consumerConfig = this.ZooKeeperBasedConsumerConfig;

            string topic1 = CurrentTestTopic + "1";
            string topic2 = CurrentTestTopic + "2";

            // first producing
            string payload1 = "kafka 1.";

            byte[] payloadData1 = Encoding.UTF8.GetBytes(payload1);
            var    msg1         = new Message(payloadData1);

            string payload2 = "kafka 2.";

            byte[] payloadData2 = Encoding.UTF8.GetBytes(payload2);
            var    msg2         = new Message(payloadData2);

            using (var producer = new SyncProducer(prodConfig))
            {
                var producerRequest1 = new ProducerRequest(topic1, 0, new List <Message> {
                    msg1
                });
                producer.Send(producerRequest1);
                var producerRequest2 = new ProducerRequest(topic2, 0, new List <Message> {
                    msg2
                });
                producer.Send(producerRequest2);
            }

            // now consuming
            var resultMessages1 = new List <Message>();
            var resultMessages2 = new List <Message>();

            using (IConsumerConnector consumerConnector = new ZookeeperConsumerConnector(consumerConfig, true))
            {
                var topicCount = new Dictionary <string, int> {
                    { topic1, 1 }, { topic2, 1 }
                };
                var messages = consumerConnector.CreateMessageStreams(topicCount);

                Assert.IsTrue(messages.ContainsKey(topic1));
                Assert.IsTrue(messages.ContainsKey(topic2));

                var sets1 = messages[topic1];
                try
                {
                    foreach (var set in sets1)
                    {
                        foreach (var message in set)
                        {
                            resultMessages1.Add(message);
                        }
                    }
                }
                catch (ConsumerTimeoutException)
                {
                    // do nothing, this is expected
                }

                var sets2 = messages[topic2];
                try
                {
                    foreach (var set in sets2)
                    {
                        foreach (var message in set)
                        {
                            resultMessages2.Add(message);
                        }
                    }
                }
                catch (ConsumerTimeoutException)
                {
                    // do nothing, this is expected
                }
            }

            Assert.AreEqual(1, resultMessages1.Count);
            Assert.AreEqual(msg1.ToString(), resultMessages1[0].ToString());

            Assert.AreEqual(1, resultMessages2.Count);
            Assert.AreEqual(msg2.ToString(), resultMessages2[0].ToString());
        }
Exemple #8
0
 public KafkaProducerException(SyncProducer syncProducer, ProducerSyncExceptionOptions options)
     : this(syncProducer)
 {
     this.options = options;
 }
Exemple #9
0
 public SyncPipeOutput(string topic, TimeSpan consumeTimeout, IStreamConfig configuration, SyncProducer producer, CancellationToken token)
 {
     this.token = token;
     topicName  = topic;
     timeout    = consumeTimeout;
     consumer   = new SyncConsumer(configuration.ToConsumerConfig($"pipe-output-{configuration.ApplicationId}-{topicName}"), producer);
     consumer.Subscribe(topicName);
 }
Exemple #10
0
        public void MaxFetchSizeBugShouldNotAppearWhenSmallFetchSizeAndSingleMessageSmallerThanFetchSize()
        {
            var prodConfig     = this.SyncProducerConfig1;
            var consumerConfig = this.ZooKeeperBasedConsumerConfig;

            consumerConfig.FetchSize          = 256;
            consumerConfig.NumberOfTries      = 1;
            consumerConfig.AutoCommitInterval = 1000;
            int    numberOfMessagesToSend = 100;
            string topic = CurrentTestTopic;

            var msgList = new List <Message>();

            using (var producer = new SyncProducer(prodConfig))
            {
                for (int i = 0; i < numberOfMessagesToSend; i++)
                {
                    string payload     = CreatePayloadByNumber(i + 100);
                    byte[] payloadData = Encoding.UTF8.GetBytes(payload);
                    var    msg         = new Message(payloadData);
                    msgList.Add(msg);
                    var producerRequest = new ProducerRequest(topic, 0, new List <Message>()
                    {
                        msg
                    });
                    producer.Send(producerRequest);
                }
            }

            Thread.Sleep(3000);

            // now consuming
            int messageNumberCounter = 0;

            using (IConsumerConnector consumerConnector = new ZookeeperConsumerConnector(consumerConfig, true))
            {
                var topicCount = new Dictionary <string, int> {
                    { topic, 1 }
                };
                var messages = consumerConnector.CreateMessageStreams(topicCount);
                var sets     = messages[topic];

                try
                {
                    foreach (var set in sets)
                    {
                        foreach (var message in set)
                        {
                            Assert.AreEqual(CreatePayloadByNumber(messageNumberCounter + 100), Encoding.UTF8.GetString(message.Payload));
                            messageNumberCounter++;
                        }
                    }
                }
                catch (ConsumerTimeoutException)
                {
                    // do nothing, this is expected
                }
            }

            Assert.AreEqual(numberOfMessagesToSend, messageNumberCounter);
        }