Constructs a request to send to Kafka.
Inheritance: AbstractRequest, IWritable
コード例 #1
0
        public void AsyncProducerSendsAndConsumerReceivesSingleSimpleMessage()
        {
            Message sourceMessage = new Message(Encoding.UTF8.GetBytes("test message"));

            var config = new AsyncProducerConfig(clientConfig);
            var producer = new AsyncProducer(config);
            var producerRequest = new ProducerRequest(CurrentTestTopic, 0, new List<Message>() { sourceMessage });

            long currentOffset = TestHelper.GetCurrentKafkaOffset(CurrentTestTopic, clientConfig);

            producer.Send(producerRequest);

            ConsumerConfig consumerConfig = new ConsumerConfig(clientConfig);
            IConsumer consumer = new Consumers.Consumer(consumerConfig);
            FetchRequest request = new FetchRequest(CurrentTestTopic, 0, currentOffset);

            BufferedMessageSet response;
            int totalWaitTimeInMiliseconds = 0;
            int waitSingle = 100;
            while (true)
            {
                Thread.Sleep(waitSingle);
                response = consumer.Fetch(request);
                if (response != null && response.Messages.Count() > 0)
                {
                    break;
                }
                else
                {
                    totalWaitTimeInMiliseconds += waitSingle;
                    if (totalWaitTimeInMiliseconds >= MaxTestWaitTimeInMiliseconds)
                    {
                        break;
                    }
                }
            }

            Assert.NotNull(response);
            Assert.AreEqual(1, response.Messages.Count());
            Message resultMessage = response.Messages.First();
            Assert.AreEqual(sourceMessage.ToString(), resultMessage.ToString());
        }
コード例 #2
0
ファイル: ProducerRequestTests.cs プロジェクト: precog/kafka
        public void WriteToValidFormat()
        {
            string topicName = "topic";
            ProducerRequest request = new ProducerRequest(
                topicName, 0, new List<Message> { new Message(new byte[10]) });

            // format = len(request) + requesttype + len(topic) + topic + partition + len(messagepack) + message
            // total byte count = (4 + 2 + 2 + 5 + 4 + 4 + 19)
            System.IO.MemoryStream ms = new MemoryStream();
            request.WriteTo(ms);

            byte[] bytes = ms.ToArray();
            Assert.IsNotNull(bytes);
            Assert.AreEqual(41, bytes.Length);

            // next 4 bytes = the length of the request
            Assert.AreEqual(37, BitConverter.ToInt32(BitWorks.ReverseBytes(bytes.Take(4).ToArray<byte>()), 0));

            // next 2 bytes = the RequestType which in this case should be Produce
            Assert.AreEqual((short)RequestTypes.Produce, BitConverter.ToInt16(BitWorks.ReverseBytes(bytes.Skip(4).Take(2).ToArray<byte>()), 0));

            // next 2 bytes = the length of the topic
            Assert.AreEqual((short)5, BitConverter.ToInt16(BitWorks.ReverseBytes(bytes.Skip(6).Take(2).ToArray<byte>()), 0));

            // next 5 bytes = the topic
            Assert.AreEqual(topicName, Encoding.ASCII.GetString(bytes.Skip(8).Take(5).ToArray<byte>()));

            // next 4 bytes = the partition
            Assert.AreEqual(0, BitConverter.ToInt32(BitWorks.ReverseBytes(bytes.Skip(13).Take(4).ToArray<byte>()), 0));

            // next 4 bytes = the length of the individual messages in the pack
            Assert.AreEqual(20, BitConverter.ToInt32(BitWorks.ReverseBytes(bytes.Skip(17).Take(4).ToArray<byte>()), 0));

            // fianl bytes = the individual messages in the pack
            Assert.AreEqual(20, bytes.Skip(21).ToArray<byte>().Length);
        }
コード例 #3
0
ファイル: SyncProducer.cs プロジェクト: precog/kafka
 /// <summary>
 /// Sends request to Kafka server synchronously
 /// </summary>
 /// <param name="request">
 /// The request.
 /// </param>
 public void Send(ProducerRequest request)
 {
     this.EnsuresNotDisposed();
     this.connection.Write(request);
 }
コード例 #4
0
ファイル: AsyncProducer.cs プロジェクト: ewhauser/kafka
 /// <summary>
 /// Sends request to Kafka server asynchronously
 /// </summary>
 /// <param name="request">
 /// The request.
 /// </param>
 public void Send(ProducerRequest request)
 {
     Guard.Assert<ArgumentNullException>(() => request != null);
     Guard.Assert<ArgumentException>(() => request.MessageSet.Messages.All(x => x.PayloadSize <= this.Config.MaxMessageSize));
     if (this.callbackHandler != null)
     {
         this.Send(request, this.callbackHandler.Handle);
     }
     else
     {
         using (var conn = new KafkaConnection(this.config.Host, this.config.Port))
         {
             conn.BeginWrite(request);
         }
     }
 }
コード例 #5
0
ファイル: AsyncProducer.cs プロジェクト: ewhauser/kafka
        /// <summary>
        /// Sends request to Kafka server asynchronously
        /// </summary>
        /// <param name="request">
        /// The request.
        /// </param>
        /// <param name="callback">
        /// The callback invoked when a request is finished being sent.
        /// </param>
        public void Send(ProducerRequest request, MessageSent<ProducerRequest> callback)
        {
            Guard.Assert<ArgumentNullException>(() => request != null);
            Guard.Assert<ArgumentNullException>(() => request.MessageSet != null);
            Guard.Assert<ArgumentNullException>(() => request.MessageSet.Messages != null);
            Guard.Assert<ArgumentException>(
                () => request.MessageSet.Messages.All(x => x.PayloadSize <= this.Config.MaxMessageSize));

            using (var conn = new KafkaConnection(this.config.Host, this.config.Port))
            {
                conn.BeginWrite(request, callback);
            }
        }
コード例 #6
0
 public void ProducerSendsMessageWithLongTopic()
 {
     Message msg = new Message(Encoding.UTF8.GetBytes("test message"));
     string topic = "ThisIsAVeryLongTopicThisIsAVeryLongTopicThisIsAVeryLongTopicThisIsAVeryLongTopicThisIsAVeryLongTopicThisIsAVeryLongTopic";
     var config = new SyncProducerConfig(clientConfig);
     var producer = new SyncProducer(config);
     var producerRequest = new ProducerRequest(topic, 0, new List<Message>() { msg });
     producer.Send(producerRequest);
 }
コード例 #7
0
        public void ProducerSendsMessage()
        {
            string payload1 = "kafka 1.";
            byte[] payloadData1 = Encoding.UTF8.GetBytes(payload1);
            Message msg1 = new Message(payloadData1);

            string payload2 = "kafka 2.";
            byte[] payloadData2 = Encoding.UTF8.GetBytes(payload2);
            Message msg2 = new Message(payloadData2);

            var config = new SyncProducerConfig(clientConfig);
            var producer = new SyncProducer(config);
            var producerRequest = new ProducerRequest(CurrentTestTopic, 0, new List<Message>() { msg1, msg2 });
            producer.Send(producerRequest);
        }
コード例 #8
0
ファイル: KafkaIntegrationTest.cs プロジェクト: precog/kafka
        public void ProducerSendsMessageWithLongTopic()
        {
            var prodConfig = this.SyncProducerConfig1;

            var msg = new Message(Encoding.UTF8.GetBytes("test message"));
            string topic = "ThisIsAVeryLongTopicThisIsAVeryLongTopicThisIsAVeryLongTopicThisIsAVeryLongTopicThisIsAVeryLongTopicThisIsAVeryLongTopic";
            using (var producer = new SyncProducer(prodConfig))
            {
                var producerRequest = new ProducerRequest(topic, 0, new List<Message> { msg });
                producer.Send(producerRequest);
            }
        }
コード例 #9
0
ファイル: KafkaIntegrationTest.cs プロジェクト: precog/kafka
        public void ProducerSendsMessage()
        {
            var prodConfig = this.SyncProducerConfig1;

            string payload1 = "kafka 1.";
            byte[] payloadData1 = Encoding.UTF8.GetBytes(payload1);
            var msg1 = new Message(payloadData1);

            string payload2 = "kafka 2.";
            byte[] payloadData2 = Encoding.UTF8.GetBytes(payload2);
            var msg2 = new Message(payloadData2);

            using (var producer = new SyncProducer(prodConfig))
            {
                var producerRequest = new ProducerRequest(CurrentTestTopic, 0, new List<Message> { msg1, msg2 });
                producer.Send(producerRequest);
            }
        }
コード例 #10
0
ファイル: KafkaIntegrationTest.cs プロジェクト: precog/kafka
        public void AsyncProducerSendsFewShortFixedMessagesInSeparateSendActions()
        {
            var prodConfig = this.AsyncProducerConfig1;

            using (var producer = new AsyncProducer(prodConfig))
            {
                var req1 = new ProducerRequest(
                    CurrentTestTopic,
                    0,
                    new List<Message> { new Message(Encoding.UTF8.GetBytes("Async Test Message 1")) });
                producer.Send(req1);

                var req2 = new ProducerRequest(
                    CurrentTestTopic,
                    0,
                    new List<Message> { new Message(Encoding.UTF8.GetBytes("Async Test Message 2")) });
                producer.Send(req2);

                var req3 = new ProducerRequest(
                    CurrentTestTopic,
                    0,
                    new List<Message> { new Message(Encoding.UTF8.GetBytes("Async Test Message 3")) });
                producer.Send(req3);
            }
        }
コード例 #11
0
ファイル: AsyncProducer.cs プロジェクト: precog/kafka
        /// <summary>
        /// Sends request to Kafka server asynchronously
        /// </summary>
        /// <param name="request">
        /// The request.
        /// </param>
        /// <param name="callback">
        /// The callback invoked when a request is finished being sent.
        /// </param>
        public void Send(ProducerRequest request, MessageSent<ProducerRequest> callback)
        {
            this.EnsuresNotDisposed();
            Guard.NotNull(request, "request");
            Guard.NotNull(request.MessageSet, "request.MessageSet");
            Guard.NotNull(request.MessageSet.Messages, "request.MessageSet.Messages");
            Guard.Assert<ArgumentException>(
                () => request.MessageSet.Messages.All(x => x.PayloadSize <= this.Config.MaxMessageSize));

            connection.BeginWrite(request, callback);
        }
コード例 #12
0
ファイル: AsyncProducer.cs プロジェクト: precog/kafka
 /// <summary>
 /// Sends request to Kafka server asynchronously
 /// </summary>
 /// <param name="request">
 /// The request.
 /// </param>
 public void Send(ProducerRequest request)
 {
     this.EnsuresNotDisposed();
     Guard.NotNull(request, "request");
     Guard.Assert<ArgumentException>(() => request.MessageSet.Messages.All(x => x.PayloadSize <= this.Config.MaxMessageSize));
     if (this.callbackHandler != null)
     {
         this.Send(request, this.callbackHandler.Handle);
     }
     else
     {
         this.connection.BeginWrite(request);
     }
 }
コード例 #13
0
ファイル: ConsumerTests.cs プロジェクト: ewhauser/kafka
        public void ConsumerConnectorConsumesTwoDifferentTopics()
        {
            string topic1 = CurrentTestTopic + "1";
            string topic2 = CurrentTestTopic + "2";

            // first producing
            string payload1 = "kafka 1.";
            byte[] payloadData1 = Encoding.UTF8.GetBytes(payload1);
            var msg1 = new Message(payloadData1);

            string payload2 = "kafka 2.";
            byte[] payloadData2 = Encoding.UTF8.GetBytes(payload2);
            var msg2 = new Message(payloadData2);

            var producerConfig = new SyncProducerConfig(clientConfig);
            var producer = new SyncProducer(producerConfig);
            var producerRequest1 = new ProducerRequest(topic1, 0, new List<Message> { msg1 });
            producer.Send(producerRequest1);
            var producerRequest2 = new ProducerRequest(topic2, 0, new List<Message> { msg2 });
            producer.Send(producerRequest2);

            // now consuming
            var config = new ConsumerConfig(clientConfig) { AutoCommit = false };
            var resultMessages1 = new List<Message>();
            var resultMessages2 = new List<Message>();
            using (IConsumerConnector consumerConnector = new ZookeeperConsumerConnector(config, true))
            {
                var topicCount = new Dictionary<string, int> { { topic1, 1 }, { topic2, 1 } };
                var messages = consumerConnector.CreateMessageStreams(topicCount);

                Assert.IsTrue(messages.ContainsKey(topic1));
                Assert.IsTrue(messages.ContainsKey(topic2));

                var sets1 = messages[topic1];
                try
                {
                    foreach (var set in sets1)
                    {
                        foreach (var message in set)
                        {
                            resultMessages1.Add(message);
                        }
                    }
                }
                catch (ConsumerTimeoutException)
                {
                    // do nothing, this is expected
                }

                var sets2 = messages[topic2];
                try
                {
                    foreach (var set in sets2)
                    {
                        foreach (var message in set)
                        {
                            resultMessages2.Add(message);
                        }
                    }
                }
                catch (ConsumerTimeoutException)
                {
                    // do nothing, this is expected
                }
            }

            Assert.AreEqual(1, resultMessages1.Count);
            Assert.AreEqual(msg1.ToString(), resultMessages1[0].ToString());

            Assert.AreEqual(1, resultMessages2.Count);
            Assert.AreEqual(msg2.ToString(), resultMessages2[0].ToString());
        }
コード例 #14
0
ファイル: ConsumerTests.cs プロジェクト: ewhauser/kafka
        public void SimpleSyncProducerSends2MessagesAndConsumerConnectorGetsThemBack()
        {
            // first producing
            string payload1 = "kafka 1.";
            byte[] payloadData1 = Encoding.UTF8.GetBytes(payload1);
            var msg1 = new Message(payloadData1);

            string payload2 = "kafka 2.";
            byte[] payloadData2 = Encoding.UTF8.GetBytes(payload2);
            var msg2 = new Message(payloadData2);

            var producerConfig = new SyncProducerConfig(clientConfig);
            var producer = new SyncProducer(producerConfig);
            var producerRequest = new ProducerRequest(CurrentTestTopic, 0, new List<Message> { msg1, msg2 });
            producer.Send(producerRequest);

            // now consuming
            var config = new ConsumerConfig(clientConfig) { AutoCommit = false };
            var resultMessages = new List<Message>();
            using (IConsumerConnector consumerConnector = new ZookeeperConsumerConnector(config, true))
            {
                var topicCount = new Dictionary<string, int> { { CurrentTestTopic, 1 } };
                var messages = consumerConnector.CreateMessageStreams(topicCount);
                var sets = messages[CurrentTestTopic];
                try
                {
                    foreach (var set in sets)
                    {
                        foreach (var message in set)
                        {
                            resultMessages.Add(message);
                        }
                    }
                }
                catch (ConsumerTimeoutException)
                {
                    // do nothing, this is expected
                }
            }

            Assert.AreEqual(2, resultMessages.Count);
            Assert.AreEqual(msg1.ToString(), resultMessages[0].ToString());
            Assert.AreEqual(msg2.ToString(), resultMessages[1].ToString());
        }
コード例 #15
0
ファイル: ConsumerTests.cs プロジェクト: ewhauser/kafka
        public void OneMessageIsSentAndReceivedThenExceptionsWhenNoMessageThenAnotherMessageIsSentAndReceived()
        {
            // first producing
            string payload1 = "kafka 1.";
            byte[] payloadData1 = Encoding.UTF8.GetBytes(payload1);
            var msg1 = new Message(payloadData1);

            var producerConfig = new SyncProducerConfig(clientConfig);
            var producer = new SyncProducer(producerConfig);
            var producerRequest = new ProducerRequest(CurrentTestTopic, 0, new List<Message> { msg1 });
            producer.Send(producerRequest);

            // now consuming
            var config = new ConsumerConfig(clientConfig) { AutoCommit = false, Timeout = 5000 };
            using (IConsumerConnector consumerConnector = new ZookeeperConsumerConnector(config, true))
            {
                var topicCount = new Dictionary<string, int> { { CurrentTestTopic, 1 } };
                var messages = consumerConnector.CreateMessageStreams(topicCount);
                var sets = messages[CurrentTestTopic];
                KafkaMessageStream myStream = sets[0];
                var enumerator = myStream.GetEnumerator();

                Assert.IsTrue(enumerator.MoveNext());
                Assert.AreEqual(msg1.ToString(), enumerator.Current.ToString());

                Assert.Throws<ConsumerTimeoutException>(() => enumerator.MoveNext());

                Assert.Throws<Exception>(() => enumerator.MoveNext()); // iterator is in failed state

                enumerator.Reset();

                // producing again
                string payload2 = "kafka 2.";
                byte[] payloadData2 = Encoding.UTF8.GetBytes(payload2);
                var msg2 = new Message(payloadData2);

                var producerRequest2 = new ProducerRequest(CurrentTestTopic, 0, new List<Message> { msg2 });
                producer.Send(producerRequest2);

                Thread.Sleep(3000);

                Assert.IsTrue(enumerator.MoveNext());
                Assert.AreEqual(msg2.ToString(), enumerator.Current.ToString());
            }
        }
コード例 #16
0
ファイル: SyncProducer.cs プロジェクト: ewhauser/kafka
 /// <summary>
 /// Sends request to Kafka server synchronously
 /// </summary>
 /// <param name="request">
 /// The request.
 /// </param>
 public void Send(ProducerRequest request)
 {
     Guard.Assert<ArgumentNullException>(() => request != null);
     using (var conn = new KafkaConnection(this.config.Host, this.config.Port))
     {
         conn.Write(request);
     }
 }