public void ProducerSendsMessageWithLongTopic() { Message msg = new Message(Encoding.UTF8.GetBytes("test message")); string topic = "ThisIsAVeryLongTopicThisIsAVeryLongTopicThisIsAVeryLongTopicThisIsAVeryLongTopicThisIsAVeryLongTopicThisIsAVeryLongTopic"; var config = new SyncProducerConfig(clientConfig); var producer = new SyncProducer(config); var producerRequest = new ProducerRequest(topic, 0, new List<Message>() { msg }); producer.Send(producerRequest); }
public void ProducerSendsAndConsumerReceivesSingleSimpleMessage() { Message sourceMessage = new Message(Encoding.UTF8.GetBytes("test message")); var config = new SyncProducerConfig(clientConfig); var producer = new SyncProducer(config); var producerRequest = new ProducerRequest(CurrentTestTopic, 0, new List<Message>() { sourceMessage }); long currentOffset = TestHelper.GetCurrentKafkaOffset(CurrentTestTopic, clientConfig); producer.Send(producerRequest); ConsumerConfig consumerConfig = new ConsumerConfig(clientConfig); IConsumer consumer = new Consumers.Consumer(consumerConfig); FetchRequest request = new FetchRequest(CurrentTestTopic, 0, currentOffset); BufferedMessageSet response; int totalWaitTimeInMiliseconds = 0; int waitSingle = 100; while (true) { Thread.Sleep(waitSingle); response = consumer.Fetch(request); if (response != null && response.Messages.Count() > 0) { break; } else { totalWaitTimeInMiliseconds += waitSingle; if (totalWaitTimeInMiliseconds >= MaxTestWaitTimeInMiliseconds) { break; } } } Assert.NotNull(response); Assert.AreEqual(1, response.Messages.Count()); Message resultMessage = response.Messages.First(); Assert.AreEqual(sourceMessage.ToString(), resultMessage.ToString()); }
public void ProducerSendsMessage() { string payload1 = "kafka 1."; byte[] payloadData1 = Encoding.UTF8.GetBytes(payload1); Message msg1 = new Message(payloadData1); string payload2 = "kafka 2."; byte[] payloadData2 = Encoding.UTF8.GetBytes(payload2); Message msg2 = new Message(payloadData2); var config = new SyncProducerConfig(clientConfig); var producer = new SyncProducer(config); var producerRequest = new ProducerRequest(CurrentTestTopic, 0, new List<Message>() { msg1, msg2 }); producer.Send(producerRequest); }
public void ProducerSendsMessageWithLongTopic() { var prodConfig = this.SyncProducerConfig1; var msg = new Message(Encoding.UTF8.GetBytes("test message")); string topic = "ThisIsAVeryLongTopicThisIsAVeryLongTopicThisIsAVeryLongTopicThisIsAVeryLongTopicThisIsAVeryLongTopicThisIsAVeryLongTopic"; using (var producer = new SyncProducer(prodConfig)) { var producerRequest = new ProducerRequest(topic, 0, new List<Message> { msg }); producer.Send(producerRequest); } }
public void ProducerSendsMessage() { var prodConfig = this.SyncProducerConfig1; string payload1 = "kafka 1."; byte[] payloadData1 = Encoding.UTF8.GetBytes(payload1); var msg1 = new Message(payloadData1); string payload2 = "kafka 2."; byte[] payloadData2 = Encoding.UTF8.GetBytes(payload2); var msg2 = new Message(payloadData2); using (var producer = new SyncProducer(prodConfig)) { var producerRequest = new ProducerRequest(CurrentTestTopic, 0, new List<Message> { msg1, msg2 }); producer.Send(producerRequest); } }
public void ConsumerConnectorConsumesTwoDifferentTopics() { string topic1 = CurrentTestTopic + "1"; string topic2 = CurrentTestTopic + "2"; // first producing string payload1 = "kafka 1."; byte[] payloadData1 = Encoding.UTF8.GetBytes(payload1); var msg1 = new Message(payloadData1); string payload2 = "kafka 2."; byte[] payloadData2 = Encoding.UTF8.GetBytes(payload2); var msg2 = new Message(payloadData2); var producerConfig = new SyncProducerConfig(clientConfig); var producer = new SyncProducer(producerConfig); var producerRequest1 = new ProducerRequest(topic1, 0, new List<Message> { msg1 }); producer.Send(producerRequest1); var producerRequest2 = new ProducerRequest(topic2, 0, new List<Message> { msg2 }); producer.Send(producerRequest2); // now consuming var config = new ConsumerConfig(clientConfig) { AutoCommit = false }; var resultMessages1 = new List<Message>(); var resultMessages2 = new List<Message>(); using (IConsumerConnector consumerConnector = new ZookeeperConsumerConnector(config, true)) { var topicCount = new Dictionary<string, int> { { topic1, 1 }, { topic2, 1 } }; var messages = consumerConnector.CreateMessageStreams(topicCount); Assert.IsTrue(messages.ContainsKey(topic1)); Assert.IsTrue(messages.ContainsKey(topic2)); var sets1 = messages[topic1]; try { foreach (var set in sets1) { foreach (var message in set) { resultMessages1.Add(message); } } } catch (ConsumerTimeoutException) { // do nothing, this is expected } var sets2 = messages[topic2]; try { foreach (var set in sets2) { foreach (var message in set) { resultMessages2.Add(message); } } } catch (ConsumerTimeoutException) { // do nothing, this is expected } } Assert.AreEqual(1, resultMessages1.Count); Assert.AreEqual(msg1.ToString(), resultMessages1[0].ToString()); Assert.AreEqual(1, resultMessages2.Count); Assert.AreEqual(msg2.ToString(), resultMessages2[0].ToString()); }
public void SimpleSyncProducerSends2MessagesAndConsumerConnectorGetsThemBack() { // first producing string payload1 = "kafka 1."; byte[] payloadData1 = Encoding.UTF8.GetBytes(payload1); var msg1 = new Message(payloadData1); string payload2 = "kafka 2."; byte[] payloadData2 = Encoding.UTF8.GetBytes(payload2); var msg2 = new Message(payloadData2); var producerConfig = new SyncProducerConfig(clientConfig); var producer = new SyncProducer(producerConfig); var producerRequest = new ProducerRequest(CurrentTestTopic, 0, new List<Message> { msg1, msg2 }); producer.Send(producerRequest); // now consuming var config = new ConsumerConfig(clientConfig) { AutoCommit = false }; var resultMessages = new List<Message>(); using (IConsumerConnector consumerConnector = new ZookeeperConsumerConnector(config, true)) { var topicCount = new Dictionary<string, int> { { CurrentTestTopic, 1 } }; var messages = consumerConnector.CreateMessageStreams(topicCount); var sets = messages[CurrentTestTopic]; try { foreach (var set in sets) { foreach (var message in set) { resultMessages.Add(message); } } } catch (ConsumerTimeoutException) { // do nothing, this is expected } } Assert.AreEqual(2, resultMessages.Count); Assert.AreEqual(msg1.ToString(), resultMessages[0].ToString()); Assert.AreEqual(msg2.ToString(), resultMessages[1].ToString()); }
public void OneMessageIsSentAndReceivedThenExceptionsWhenNoMessageThenAnotherMessageIsSentAndReceived() { // first producing string payload1 = "kafka 1."; byte[] payloadData1 = Encoding.UTF8.GetBytes(payload1); var msg1 = new Message(payloadData1); var producerConfig = new SyncProducerConfig(clientConfig); var producer = new SyncProducer(producerConfig); var producerRequest = new ProducerRequest(CurrentTestTopic, 0, new List<Message> { msg1 }); producer.Send(producerRequest); // now consuming var config = new ConsumerConfig(clientConfig) { AutoCommit = false, Timeout = 5000 }; using (IConsumerConnector consumerConnector = new ZookeeperConsumerConnector(config, true)) { var topicCount = new Dictionary<string, int> { { CurrentTestTopic, 1 } }; var messages = consumerConnector.CreateMessageStreams(topicCount); var sets = messages[CurrentTestTopic]; KafkaMessageStream myStream = sets[0]; var enumerator = myStream.GetEnumerator(); Assert.IsTrue(enumerator.MoveNext()); Assert.AreEqual(msg1.ToString(), enumerator.Current.ToString()); Assert.Throws<ConsumerTimeoutException>(() => enumerator.MoveNext()); Assert.Throws<Exception>(() => enumerator.MoveNext()); // iterator is in failed state enumerator.Reset(); // producing again string payload2 = "kafka 2."; byte[] payloadData2 = Encoding.UTF8.GetBytes(payload2); var msg2 = new Message(payloadData2); var producerRequest2 = new ProducerRequest(CurrentTestTopic, 0, new List<Message> { msg2 }); producer.Send(producerRequest2); Thread.Sleep(3000); Assert.IsTrue(enumerator.MoveNext()); Assert.AreEqual(msg2.ToString(), enumerator.Current.ToString()); } }