public void ProducerSends1Message() { var prodConfig = this.ConfigBasedSyncProdConfig; int totalWaitTimeInMiliseconds = 0; int waitSingle = 100; var originalMessage = new Message(Encoding.UTF8.GetBytes("TestData")); var multipleBrokersHelper = new TestMultipleBrokersHelper(CurrentTestTopic); multipleBrokersHelper.GetCurrentOffsets( new[] { this.SyncProducerConfig1, this.SyncProducerConfig2, this.SyncProducerConfig3 }); using (var producer = new Producer(prodConfig)) { var producerData = new ProducerData<string, Message>( CurrentTestTopic, new List<Message> { originalMessage }); producer.Send(producerData); Thread.Sleep(waitSingle); } while ( !multipleBrokersHelper.CheckIfAnyBrokerHasChanged( new[] { this.SyncProducerConfig1, this.SyncProducerConfig2, this.SyncProducerConfig3 })) { totalWaitTimeInMiliseconds += waitSingle; Thread.Sleep(waitSingle); if (totalWaitTimeInMiliseconds > this.maxTestWaitTimeInMiliseconds) { Assert.Fail("None of the brokers changed their offset after sending a message"); } } totalWaitTimeInMiliseconds = 0; var consumerConfig = new ConsumerConfiguration( multipleBrokersHelper.BrokerThatHasChanged.Host, multipleBrokersHelper.BrokerThatHasChanged.Port); IConsumer consumer = new Consumer(consumerConfig); var request1 = new FetchRequest(CurrentTestTopic, multipleBrokersHelper.PartitionThatHasChanged, multipleBrokersHelper.OffsetFromBeforeTheChange); BufferedMessageSet response; while (true) { Thread.Sleep(waitSingle); response = consumer.Fetch(request1); if (response != null && response.Messages.Count() > 0) { break; } totalWaitTimeInMiliseconds += waitSingle; if (totalWaitTimeInMiliseconds >= this.maxTestWaitTimeInMiliseconds) { break; } } Assert.NotNull(response); Assert.AreEqual(1, response.Messages.Count()); Assert.AreEqual(originalMessage.ToString(), response.Messages.First().ToString()); }
public void ProducerSends1Message() { int totalWaitTimeInMiliseconds = 0; int waitSingle = 100; var originalMessage = new Message(Encoding.UTF8.GetBytes("TestData")); var multipleBrokersHelper = new TestMultipleBrokersHelper(CurrentTestTopic); multipleBrokersHelper.GetCurrentOffsets(); var producerConfig = new ProducerConfig(clientConfig); var mockPartitioner = new MockAlwaysZeroPartitioner(); using (var producer = new Producer<string, Message>(producerConfig, mockPartitioner, new DefaultEncoder(), null)) { var producerData = new ProducerData<string, Message>( CurrentTestTopic, "somekey", new List<Message> { originalMessage }); producer.Send(producerData); Thread.Sleep(waitSingle); while (!multipleBrokersHelper.CheckIfAnyBrokerHasChanged()) { totalWaitTimeInMiliseconds += waitSingle; Thread.Sleep(waitSingle); if (totalWaitTimeInMiliseconds > this.maxTestWaitTimeInMiliseconds) { Assert.Fail("None of the brokers changed their offset after sending a message"); } } totalWaitTimeInMiliseconds = 0; var consumerConfig = new ConsumerConfig(clientConfig) { Host = multipleBrokersHelper.BrokerThatHasChanged.Address, Port = multipleBrokersHelper.BrokerThatHasChanged.Port }; IConsumer consumer = new Consumers.Consumer(consumerConfig); var request = new FetchRequest(CurrentTestTopic, 0, multipleBrokersHelper.OffsetFromBeforeTheChange); BufferedMessageSet response; while (true) { Thread.Sleep(waitSingle); response = consumer.Fetch(request); if (response != null && response.Messages.Count() > 0) { break; } totalWaitTimeInMiliseconds += waitSingle; if (totalWaitTimeInMiliseconds >= this.maxTestWaitTimeInMiliseconds) { break; } } Assert.NotNull(response); Assert.AreEqual(1, response.Messages.Count()); Assert.AreEqual(originalMessage.ToString(), response.Messages.First().ToString()); } }
public void ProducerSendsAndConsumerReceivesMultiRequest() { string testTopic1 = CurrentTestTopic + "1"; string testTopic2 = CurrentTestTopic + "2"; string testTopic3 = CurrentTestTopic + "3"; Message sourceMessage1 = new Message(Encoding.UTF8.GetBytes("1: TestMessage")); Message sourceMessage2 = new Message(Encoding.UTF8.GetBytes("2: TestMessage")); Message sourceMessage3 = new Message(Encoding.UTF8.GetBytes("3: TestMessage")); Message sourceMessage4 = new Message(Encoding.UTF8.GetBytes("4: TestMessage")); List<ProducerRequest> requests = new List<ProducerRequest> { new ProducerRequest(testTopic1, 0, new List<Message> { sourceMessage1 }), new ProducerRequest(testTopic1, 0, new List<Message> { sourceMessage2 }), new ProducerRequest(testTopic2, 0, new List<Message> { sourceMessage3 }), new ProducerRequest(testTopic3, 0, new List<Message> { sourceMessage4 }) }; var config = new SyncProducerConfig(clientConfig); var producer = new SyncProducer(config); long currentOffset1 = TestHelper.GetCurrentKafkaOffset(testTopic1, clientConfig); long currentOffset2 = TestHelper.GetCurrentKafkaOffset(testTopic2, clientConfig); long currentOffset3 = TestHelper.GetCurrentKafkaOffset(testTopic3, clientConfig); producer.MultiSend(requests); ConsumerConfig consumerConfig = new ConsumerConfig(clientConfig); IConsumer consumer = new Consumers.Consumer(consumerConfig); MultiFetchRequest request = new MultiFetchRequest(new List<FetchRequest> { new FetchRequest(testTopic1, 0, currentOffset1), new FetchRequest(testTopic2, 0, currentOffset2), new FetchRequest(testTopic3, 0, currentOffset3) }); IList<BufferedMessageSet> messageSets; int totalWaitTimeInMiliseconds = 0; int waitSingle = 100; while (true) { Thread.Sleep(waitSingle); messageSets = consumer.MultiFetch(request); if (messageSets.Count > 2 && messageSets[0].Messages.Count() > 0 && messageSets[1].Messages.Count() > 0 && messageSets[2].Messages.Count() > 0) { break; } else { totalWaitTimeInMiliseconds += waitSingle; if (totalWaitTimeInMiliseconds >= MaxTestWaitTimeInMiliseconds) { break; } } } Assert.AreEqual(3, messageSets.Count); Assert.AreEqual(2, messageSets[0].Messages.Count()); Assert.AreEqual(1, messageSets[1].Messages.Count()); Assert.AreEqual(1, messageSets[2].Messages.Count()); Assert.AreEqual(sourceMessage1.ToString(), messageSets[0].Messages.First().ToString()); Assert.AreEqual(sourceMessage2.ToString(), messageSets[0].Messages.Skip(1).First().ToString()); Assert.AreEqual(sourceMessage3.ToString(), messageSets[1].Messages.First().ToString()); Assert.AreEqual(sourceMessage4.ToString(), messageSets[2].Messages.First().ToString()); }
public void AsyncProducerSendsAndConsumerReceivesSingleSimpleMessage() { Message sourceMessage = new Message(Encoding.UTF8.GetBytes("test message")); var config = new AsyncProducerConfig(clientConfig); var producer = new AsyncProducer(config); var producerRequest = new ProducerRequest(CurrentTestTopic, 0, new List<Message>() { sourceMessage }); long currentOffset = TestHelper.GetCurrentKafkaOffset(CurrentTestTopic, clientConfig); producer.Send(producerRequest); ConsumerConfig consumerConfig = new ConsumerConfig(clientConfig); IConsumer consumer = new Consumers.Consumer(consumerConfig); FetchRequest request = new FetchRequest(CurrentTestTopic, 0, currentOffset); BufferedMessageSet response; int totalWaitTimeInMiliseconds = 0; int waitSingle = 100; while (true) { Thread.Sleep(waitSingle); response = consumer.Fetch(request); if (response != null && response.Messages.Count() > 0) { break; } else { totalWaitTimeInMiliseconds += waitSingle; if (totalWaitTimeInMiliseconds >= MaxTestWaitTimeInMiliseconds) { break; } } } Assert.NotNull(response); Assert.AreEqual(1, response.Messages.Count()); Message resultMessage = response.Messages.First(); Assert.AreEqual(sourceMessage.ToString(), resultMessage.ToString()); }
public void ZkAwareProducerSends3Messages() { var prodConfig = this.ZooKeeperBasedSyncProdConfig; int totalWaitTimeInMiliseconds = 0; int waitSingle = 100; var originalMessage1 = new Message(Encoding.UTF8.GetBytes("TestData1")); var originalMessage2 = new Message(Encoding.UTF8.GetBytes("TestData2")); var originalMessage3 = new Message(Encoding.UTF8.GetBytes("TestData3")); var originalMessageList = new List<Message> { originalMessage1, originalMessage2, originalMessage3 }; var multipleBrokersHelper = new TestMultipleBrokersHelper(CurrentTestTopic); multipleBrokersHelper.GetCurrentOffsets(new[] { this.SyncProducerConfig1, this.SyncProducerConfig2, this.SyncProducerConfig3 }); var mockPartitioner = new MockAlwaysZeroPartitioner(); using (var producer = new Producer<string, Message>(prodConfig, mockPartitioner, new DefaultEncoder())) { var producerData = new ProducerData<string, Message>(CurrentTestTopic, "somekey", originalMessageList); producer.Send(producerData); while (!multipleBrokersHelper.CheckIfAnyBrokerHasChanged(new[] { this.SyncProducerConfig1, this.SyncProducerConfig2, this.SyncProducerConfig3 })) { totalWaitTimeInMiliseconds += waitSingle; Thread.Sleep(waitSingle); if (totalWaitTimeInMiliseconds > this.maxTestWaitTimeInMiliseconds) { Assert.Fail("None of the brokers changed their offset after sending a message"); } } totalWaitTimeInMiliseconds = 0; var consumerConfig = new ConsumerConfiguration( multipleBrokersHelper.BrokerThatHasChanged.Host, multipleBrokersHelper.BrokerThatHasChanged.Port); IConsumer consumer = new Consumer(consumerConfig); var request = new FetchRequest(CurrentTestTopic, 0, multipleBrokersHelper.OffsetFromBeforeTheChange); BufferedMessageSet response; while (true) { Thread.Sleep(waitSingle); response = consumer.Fetch(request); if (response != null && response.Messages.Count() > 2) { break; } totalWaitTimeInMiliseconds += waitSingle; if (totalWaitTimeInMiliseconds >= this.maxTestWaitTimeInMiliseconds) { break; } } Assert.NotNull(response); Assert.AreEqual(3, response.Messages.Count()); Assert.AreEqual(originalMessage1.ToString(), response.Messages.First().ToString()); Assert.AreEqual(originalMessage2.ToString(), response.Messages.Skip(1).First().ToString()); Assert.AreEqual(originalMessage3.ToString(), response.Messages.Skip(2).First().ToString()); } }
public void ConsumerConnectorConsumesTwoDifferentTopics() { string topic1 = CurrentTestTopic + "1"; string topic2 = CurrentTestTopic + "2"; // first producing string payload1 = "kafka 1."; byte[] payloadData1 = Encoding.UTF8.GetBytes(payload1); var msg1 = new Message(payloadData1); string payload2 = "kafka 2."; byte[] payloadData2 = Encoding.UTF8.GetBytes(payload2); var msg2 = new Message(payloadData2); var producerConfig = new SyncProducerConfig(clientConfig); var producer = new SyncProducer(producerConfig); var producerRequest1 = new ProducerRequest(topic1, 0, new List<Message> { msg1 }); producer.Send(producerRequest1); var producerRequest2 = new ProducerRequest(topic2, 0, new List<Message> { msg2 }); producer.Send(producerRequest2); // now consuming var config = new ConsumerConfig(clientConfig) { AutoCommit = false }; var resultMessages1 = new List<Message>(); var resultMessages2 = new List<Message>(); using (IConsumerConnector consumerConnector = new ZookeeperConsumerConnector(config, true)) { var topicCount = new Dictionary<string, int> { { topic1, 1 }, { topic2, 1 } }; var messages = consumerConnector.CreateMessageStreams(topicCount); Assert.IsTrue(messages.ContainsKey(topic1)); Assert.IsTrue(messages.ContainsKey(topic2)); var sets1 = messages[topic1]; try { foreach (var set in sets1) { foreach (var message in set) { resultMessages1.Add(message); } } } catch (ConsumerTimeoutException) { // do nothing, this is expected } var sets2 = messages[topic2]; try { foreach (var set in sets2) { foreach (var message in set) { resultMessages2.Add(message); } } } catch (ConsumerTimeoutException) { // do nothing, this is expected } } Assert.AreEqual(1, resultMessages1.Count); Assert.AreEqual(msg1.ToString(), resultMessages1[0].ToString()); Assert.AreEqual(1, resultMessages2.Count); Assert.AreEqual(msg2.ToString(), resultMessages2[0].ToString()); }
public void SimpleSyncProducerSends2MessagesAndConsumerConnectorGetsThemBack() { // first producing string payload1 = "kafka 1."; byte[] payloadData1 = Encoding.UTF8.GetBytes(payload1); var msg1 = new Message(payloadData1); string payload2 = "kafka 2."; byte[] payloadData2 = Encoding.UTF8.GetBytes(payload2); var msg2 = new Message(payloadData2); var producerConfig = new SyncProducerConfig(clientConfig); var producer = new SyncProducer(producerConfig); var producerRequest = new ProducerRequest(CurrentTestTopic, 0, new List<Message> { msg1, msg2 }); producer.Send(producerRequest); // now consuming var config = new ConsumerConfig(clientConfig) { AutoCommit = false }; var resultMessages = new List<Message>(); using (IConsumerConnector consumerConnector = new ZookeeperConsumerConnector(config, true)) { var topicCount = new Dictionary<string, int> { { CurrentTestTopic, 1 } }; var messages = consumerConnector.CreateMessageStreams(topicCount); var sets = messages[CurrentTestTopic]; try { foreach (var set in sets) { foreach (var message in set) { resultMessages.Add(message); } } } catch (ConsumerTimeoutException) { // do nothing, this is expected } } Assert.AreEqual(2, resultMessages.Count); Assert.AreEqual(msg1.ToString(), resultMessages[0].ToString()); Assert.AreEqual(msg2.ToString(), resultMessages[1].ToString()); }
public void OneMessageIsSentAndReceivedThenExceptionsWhenNoMessageThenAnotherMessageIsSentAndReceived() { // first producing string payload1 = "kafka 1."; byte[] payloadData1 = Encoding.UTF8.GetBytes(payload1); var msg1 = new Message(payloadData1); var producerConfig = new SyncProducerConfig(clientConfig); var producer = new SyncProducer(producerConfig); var producerRequest = new ProducerRequest(CurrentTestTopic, 0, new List<Message> { msg1 }); producer.Send(producerRequest); // now consuming var config = new ConsumerConfig(clientConfig) { AutoCommit = false, Timeout = 5000 }; using (IConsumerConnector consumerConnector = new ZookeeperConsumerConnector(config, true)) { var topicCount = new Dictionary<string, int> { { CurrentTestTopic, 1 } }; var messages = consumerConnector.CreateMessageStreams(topicCount); var sets = messages[CurrentTestTopic]; KafkaMessageStream myStream = sets[0]; var enumerator = myStream.GetEnumerator(); Assert.IsTrue(enumerator.MoveNext()); Assert.AreEqual(msg1.ToString(), enumerator.Current.ToString()); Assert.Throws<ConsumerTimeoutException>(() => enumerator.MoveNext()); Assert.Throws<Exception>(() => enumerator.MoveNext()); // iterator is in failed state enumerator.Reset(); // producing again string payload2 = "kafka 2."; byte[] payloadData2 = Encoding.UTF8.GetBytes(payload2); var msg2 = new Message(payloadData2); var producerRequest2 = new ProducerRequest(CurrentTestTopic, 0, new List<Message> { msg2 }); producer.Send(producerRequest2); Thread.Sleep(3000); Assert.IsTrue(enumerator.MoveNext()); Assert.AreEqual(msg2.ToString(), enumerator.Current.ToString()); } }
public void WriteToValidSequenceForDefaultConstructor() { byte[] messageBytes = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9 }; Message message = new Message(messageBytes); MemoryStream ms = new MemoryStream(); message.WriteTo(ms); Assert.AreEqual(1, ms.ToArray()[MagicNumberPartOffset]); // default magic number should be 1 byte[] checksumPart = new byte[ChecksumPartLength]; Array.Copy(ms.ToArray(), ChecksumPartOffset, checksumPart, 0, ChecksumPartLength); Assert.AreEqual(Crc32Hasher.Compute(messageBytes), checksumPart); message.ToString(); byte[] dataPart = new byte[messageBytes.Length]; Array.Copy(ms.ToArray(), DataPartOffset, dataPart, 0, messageBytes.Length); Assert.AreEqual(messageBytes, dataPart); }