public void TestMessageSizeTooLarge() { var props = TestUtils.GetSyncProducerConfig(this.Configs[0].Port); var producer = new SyncProducer(props); AdminUtils.CreateTopic(this.ZkClient, "test", 1, 1, new Dictionary <string, string>()); TestUtils.WaitUntilLeaderIsElectedOrChanged(this.ZkClient, "test", 0, 500); TestUtils.WaitUntilMetadataIsPropagated(Servers, "test", 0, 2000); var message1 = new Message(new byte[Configs[0].MessageMaxBytes + 1]); var messageSet1 = new ByteBufferMessageSet(CompressionCodecs.NoCompressionCodec, new List <Message> { message1 }); var response1 = producer.Send(TestUtils.ProduceRequest("test", 0, messageSet1, acks: 1)); Assert.Equal(1, response1.Status.Count(kvp => kvp.Value.Error != ErrorMapping.NoError)); Assert.Equal(ErrorMapping.MessageSizeTooLargeCode, response1.Status[new TopicAndPartition("test", 0)].Error); Assert.Equal(-1L, response1.Status[new TopicAndPartition("test", 0)].Offset); var safeSize = Configs[0].MessageMaxBytes - Message.MessageOverhead - MessageSet.LogOverhead - 1; var message2 = new Message(new byte[safeSize]); var messageSet2 = new ByteBufferMessageSet( CompressionCodecs.NoCompressionCodec, new List <Message> { message2 }); var response2 = producer.Send(TestUtils.ProduceRequest("test", 0, messageSet2, acks: 1)); Assert.Equal(0, response2.Status.Count(kvp => kvp.Value.Error != ErrorMapping.NoError)); Assert.Equal(ErrorMapping.NoError, response2.Status[new TopicAndPartition("test", 0)].Error); Assert.Equal(0, response2.Status[new TopicAndPartition("test", 0)].Offset); }
public void TestGetAllTopicMetadata() { // create topic var topic1 = "testGetAllTopicMetadata1"; var topic2 = "testGetAllTopicMetadata2"; AdminUtils.CreateTopic(this.ZkClient, topic1, 1, 1, new Dictionary <string, string>()); AdminUtils.CreateTopic(this.ZkClient, topic2, 1, 1, new Dictionary <string, string>()); // wait for leader to be elected for both topics TestUtils.WaitUntilMetadataIsPropagated(this.Servers, topic1, 0, 1000); TestUtils.WaitUntilMetadataIsPropagated(this.Servers, topic2, 0, 1000); // issue metadata request with empty list of topics var topicsMetadata = ClientUtils.FetchTopicMetadata( new HashSet <string>(), this.Brokers, "TopicMetadataTest-testGetAllTopicMetadata", 2000).TopicsMetadata; Assert.Equal(ErrorMapping.NoError, topicsMetadata.First().ErrorCode); Assert.Equal(2, topicsMetadata.Count); Assert.Equal(ErrorMapping.NoError, topicsMetadata.First().PartitionsMetadata.First().ErrorCode); Assert.Equal(ErrorMapping.NoError, topicsMetadata.Last().PartitionsMetadata.First().ErrorCode); var partitionMetadataTopic1 = topicsMetadata.First().PartitionsMetadata; var partitionMetadataTopic2 = topicsMetadata.Last().PartitionsMetadata; Assert.Equal(1, partitionMetadataTopic1.Count); Assert.Equal(0, partitionMetadataTopic1.First().PartitionId); Assert.Equal(1, partitionMetadataTopic1.First().Replicas.Count()); Assert.Equal(1, partitionMetadataTopic2.Count); Assert.Equal(0, partitionMetadataTopic2.First().PartitionId); Assert.Equal(1, partitionMetadataTopic2.First().Replicas.Count()); }
public void TestSendNullMessage() { var config = new ProducerConfig { Serializer = typeof(StringEncoder).AssemblyQualifiedName, KeySerializer = typeof(StringEncoder).AssemblyQualifiedName, PartitionerClass = typeof(StaticPartitioner).AssemblyQualifiedName, Brokers = TestUtils.GetBrokerListFromConfigs( new List <TempKafkaConfig> { this.config1, this.config2 }), }; var producer = new Producer <string, string>(config); try { // create topic AdminUtils.CreateTopic(this.ZkClient, "new-topic", 2, 1, new Dictionary <string, string>()); TestUtils.WaitUntilLeaderIsElectedOrChanged(this.ZkClient, "new-topic", 0, 500); producer.Send(new KeyedMessage <string, string>("new-topic", "key", null)); } finally { producer.Dispose(); } }
public void CreateSimpleTopicsAndAwaitLeader(ZkClient zkClient, List <string> topics, int brokerId) { foreach (var topic in topics) { AdminUtils.CreateTopic(zkClient, topic, 1, 1, new Dictionary <string, string>()); TestUtils.WaitUntilLeaderIsElectedOrChanged(this.ZkClient, topic, 0, 500); } }
public void TestConsumerEmptyTopic() { var newTopic = "new-topic"; AdminUtils.CreateTopic(this.ZkClient, newTopic, 1, 1, new Dictionary <string, string>()); TestUtils.WaitUntilMetadataIsPropagated(this.Servers, newTopic, 0, 1000); TestUtils.WaitUntilLeaderIsElectedOrChanged(this.ZkClient, newTopic, 0, 500); var fetchResponse = Consumer.Fetch(new FetchRequestBuilder().AddFetch(newTopic, 0, 0, 10000).Build()); Assert.False(fetchResponse.MessageSet(newTopic, 0).Iterator().HasNext()); }
public void TestProduceCorrectlyReceivesResponse() { var props = TestUtils.GetSyncProducerConfig(this.Configs[0].Port); var producer = new SyncProducer(props); var messages = new ByteBufferMessageSet( CompressionCodecs.NoCompressionCodec, new List <Message> { new Message(this.messageBytes) }); // #1 - test that we get an error when partition does not belong to broker in response var request = TestUtils.ProduceRequestWithAcks( new List <string> { "topic1", "topic2", "topic3" }, new List <int> { 0 }, messages, 1); var response = producer.Send(request); Assert.NotNull(response); Assert.Equal(request.CorrelationId, response.CorrelationId); Assert.Equal(3, response.Status.Count); foreach (var responseStatus in response.Status.Values) { Assert.Equal(ErrorMapping.UnknownTopicOrPartitionCode, responseStatus.Error); Assert.Equal(-1L, responseStatus.Offset); } // #2 - test that we get correct offsets when partition is owned by broker AdminUtils.CreateTopic(this.ZkClient, "topic1", 1, 1, new Dictionary <string, string>()); AdminUtils.CreateTopic(this.ZkClient, "topic3", 1, 1, new Dictionary <string, string>()); TestUtils.WaitUntilLeaderIsElectedOrChanged(this.ZkClient, "topic3", 0, 5000); TestUtils.WaitUntilMetadataIsPropagated(Servers, "topic3", 0, 2000); var response2 = producer.Send(request); Assert.NotNull(response2); Assert.Equal(request.CorrelationId, response2.CorrelationId); Assert.Equal(3, response2.Status.Count); // the first and last message should have been accepted by broker Assert.Equal(ErrorMapping.NoError, response2.Status[new TopicAndPartition("topic1", 0)].Error); Assert.Equal(ErrorMapping.NoError, response2.Status[new TopicAndPartition("topic3", 0)].Error); Assert.Equal(0, response2.Status[new TopicAndPartition("topic1", 0)].Offset); Assert.Equal(0, response2.Status[new TopicAndPartition("topic3", 0)].Offset); // the middle message should have been rejected because broker doesn't lead partition Assert.Equal(ErrorMapping.UnknownTopicOrPartitionCode, response2.Status[new TopicAndPartition("topic2", 0)].Error); Assert.Equal(-1L, response2.Status[new TopicAndPartition("topic2", 0)].Offset); }
public void TestLeaderSelectionForPartition() { var zkClient = new ZkClient(this.zookeeperConnect, 6000, 30000, new ZkStringSerializer()); // create topic topic1 with 1 partition on broker 0 AdminUtils.CreateTopic(zkClient, Topic, 1, 1, new Dictionary <string, string>()); TestUtils.WaitUntilMetadataIsPropagated(this.Servers, Topic, 0, 3000); var sentMessages1 = this.SendMessages( Configs.First(), nMessages, "batch1", CompressionCodecs.NoCompressionCodec, 1); TestUtils.WaitUntilMetadataIsPropagated(this.Servers, Topic, 0, 1000); // create a consuemr var consumerConfig1 = TestUtils.CreateConsumerProperties(ZkConnect, Group, Consumer1); var zkConsumerConnector1 = new ZookeeperConsumerConnector(consumerConfig1); var topicMessageStreams1 = zkConsumerConnector1.CreateMessageStreams( new Dictionary <string, int> { { Topic, 1 } }, new StringDecoder(), new StringDecoder()); var topicRegistry = zkConsumerConnector1.TopicRegistry; Assert.Equal(1, topicRegistry.Select(x => x.Key).Count()); Assert.Equal(Topic, topicRegistry.Select(x => x.Key).First()); var topicsAndPartitionsInRegistry = topicRegistry.Select(x => Tuple.Create(x.Key, x.Value.Select(p => p.Value))).ToList(); var brokerPartition = topicsAndPartitionsInRegistry.First().Item2.First(); Assert.Equal(0, brokerPartition.PartitionId); // also check partition ownership var actual_1 = this.GetZKChildrenValues(this.dirs.ConsumerOwnerDir); var expected_1 = new List <Tuple <string, string> > { Tuple.Create("0", "group1_consumer1-0"), }; Assert.Equal(expected_1, actual_1); var receivedMessages1 = this.GetMessages(nMessages, topicMessageStreams1); Assert.Equal(sentMessages1, receivedMessages1); zkConsumerConnector1.Shutdown(); zkClient.Dispose(); }
public void TestTopicMetadataRequest() { // create topic var topic = "test"; AdminUtils.CreateTopic(this.ZkClient, topic, 1, 1, new Dictionary <string, string>()); // create a topic metadata request var topicMetadataRequest = new TopicMetadataRequest(new List <string> { topic }, 0); var serializedMetadataRequest = ByteBuffer.Allocate(topicMetadataRequest.SizeInBytes + 2); topicMetadataRequest.WriteTo(serializedMetadataRequest); serializedMetadataRequest.Rewind(); var deserializedMetadataRequest = TopicMetadataRequest.ReadFrom(serializedMetadataRequest); Assert.Equal(topicMetadataRequest, deserializedMetadataRequest); }
public void TestMessageSizeTooLargeWithAckZero() { var props = TestUtils.GetSyncProducerConfig(this.Configs[0].Port); props.RequestRequiredAcks = 0; var producer = new SyncProducer(props); AdminUtils.CreateTopic(this.ZkClient, "test", 1, 1, new Dictionary <string, string>()); TestUtils.WaitUntilLeaderIsElectedOrChanged(this.ZkClient, "test", 0, 500); // This message will be dropped silently since message size too large. producer.Send( TestUtils.ProduceRequest( "test", 0, new ByteBufferMessageSet( CompressionCodecs.NoCompressionCodec, new List <Message> { new Message(new byte[Configs[0].MessageMaxBytes + 1]) }))); // Send another message whose size is large enough to exceed the buffer size so // the socket buffer will be flushed immediately; // this send should fail since the socket has been closed try { producer.Send( TestUtils.ProduceRequest( "test", 0, new ByteBufferMessageSet( CompressionCodecs.NoCompressionCodec, new List <Message> { new Message(new byte[Configs[0].MessageMaxBytes + 1]) }))); } catch (IOException) { } }
public void TestBasicTopicMetadata() { // create topic var topic = "test"; AdminUtils.CreateTopic(this.ZkClient, topic, 1, 1, new Dictionary <string, string>()); TestUtils.WaitUntilMetadataIsPropagated(this.Servers, topic, 0, 1000); TestUtils.WaitUntilLeaderIsElectedOrChanged(this.ZkClient, topic, 0, 1000); var topicsMetadata = ClientUtils.FetchTopicMetadata( new HashSet <string> { topic }, this.Brokers, "TopicMetadataTest-testBasicTopicMetadata", 2000) .TopicsMetadata; Assert.Equal(ErrorMapping.NoError, topicsMetadata.First().ErrorCode); Assert.Equal(ErrorMapping.NoError, topicsMetadata.First().PartitionsMetadata.First().ErrorCode); Assert.Equal(1, topicsMetadata.Count); Assert.Equal("test", topicsMetadata.First().Topic); var partitionMetadata = topicsMetadata.First().PartitionsMetadata; Assert.Equal(1, partitionMetadata.Count); Assert.Equal(0, partitionMetadata.First().PartitionId); Assert.Equal(1, partitionMetadata.First().Replicas.Count()); }
public void TestSendToNewTopic() { var producerConfig1 = new ProducerConfig { Serializer = typeof(StringEncoder).AssemblyQualifiedName, KeySerializer = typeof(StringEncoder).AssemblyQualifiedName, PartitionerClass = typeof(StaticPartitioner).AssemblyQualifiedName, Brokers = TestUtils.GetBrokerListFromConfigs( new List <TempKafkaConfig> { this.config1, this.config2 }), RequestRequiredAcks = 2, RequestTimeoutMs = 1000 }; var producerConfig2 = new ProducerConfig { Serializer = typeof(StringEncoder).AssemblyQualifiedName, KeySerializer = typeof(StringEncoder).AssemblyQualifiedName, PartitionerClass = typeof(StaticPartitioner).AssemblyQualifiedName, Brokers = TestUtils.GetBrokerListFromConfigs( new List <TempKafkaConfig> { this.config1, this.config2 }), RequestRequiredAcks = 3, RequestTimeoutMs = 1000 }; var topic = "new-topic"; // create topic with 1 partition and await leadership AdminUtils.CreateTopic(this.ZkClient, topic, 1, 2, new Dictionary <string, string>()); TestUtils.WaitUntilMetadataIsPropagated(this.servers, topic, 0, 1000); TestUtils.WaitUntilLeaderIsElectedOrChanged(this.ZkClient, topic, 0, 500); var producer1 = new Producer <string, string>(producerConfig1); var producer2 = new Producer <string, string>(producerConfig2); // Available partition ids should be 0. producer1.Send(new KeyedMessage <string, string>(topic, "test", "test1")); producer1.Send(new KeyedMessage <string, string>(topic, "test", "test2")); // get the leader var leaderOpt = ZkUtils.GetLeaderForPartition(ZkClient, topic, 0); Assert.True(leaderOpt.HasValue); var leader = leaderOpt.Value; var messageSet = (leader == this.config1.BrokerId) ? this.consumer1.Fetch(new FetchRequestBuilder().AddFetch(topic, 0, 0, 10000).Build()) .MessageSet("new-topic", 0) .Iterator() .ToEnumerable() .ToList() : this.consumer2.Fetch(new FetchRequestBuilder().AddFetch(topic, 0, 0, 10000).Build()) .MessageSet("new-topic", 0) .Iterator() .ToEnumerable() .ToList(); Assert.Equal(2, messageSet.Count()); Assert.Equal(new Message(Encoding.UTF8.GetBytes("test1"), Encoding.UTF8.GetBytes("test")), messageSet[0].Message); Assert.Equal(new Message(Encoding.UTF8.GetBytes("test2"), Encoding.UTF8.GetBytes("test")), messageSet[1].Message); producer1.Dispose(); try { producer2.Send(new KeyedMessage <string, string>(topic, "test", "test2")); Assert.False(true, "Should have timed out for 3 acks."); } catch (FailedToSendMessageException) { } finally { producer2.Dispose(); } }
public void TestUpdateBrokerPartitionInfo() { var topic = "new-topic"; AdminUtils.CreateTopic(this.ZkClient, topic, 1, 2, new Dictionary <string, string>()); // wait until the update metadata request for new topic reaches all servers TestUtils.WaitUntilMetadataIsPropagated(this.servers, topic, 0, 3000); TestUtils.WaitUntilLeaderIsElectedOrChanged(this.ZkClient, topic, 0, 500); var producerConfig1 = new ProducerConfig(); producerConfig1.Brokers = new List <BrokerConfiguration> { new BrokerConfiguration { BrokerId = 0, Host = "localhost", Port = 18203 }, new BrokerConfiguration { BrokerId = 1, Host = "localhost", Port = 18204 } }; producerConfig1.KeySerializer = typeof(StringEncoder).AssemblyQualifiedName; producerConfig1.Serializer = typeof(StringEncoder).AssemblyQualifiedName; var producer1 = new Producer <string, string>(producerConfig1); try { producer1.Send(new KeyedMessage <string, string>(topic, "test", "test1")); Assert.False(true, "Test should fail because the broker list provided are not valid"); } catch (FailedToSendMessageException) { // ok } finally { producer1.Dispose(); } var producerConfig2 = new ProducerConfig(); producerConfig2.Brokers = new List <BrokerConfiguration> { new BrokerConfiguration { BrokerId = 0, Host = "localhost", Port = 18203 }, new BrokerConfiguration { BrokerId = 1, Host = "localhost", Port = this.port1 } }; producerConfig2.KeySerializer = typeof(StringEncoder).AssemblyQualifiedName; producerConfig2.Serializer = typeof(StringEncoder).AssemblyQualifiedName; var producer2 = new Producer <string, string>(producerConfig2); try { producer2.Send(new KeyedMessage <string, string>(topic, "test", "test1")); } finally { producer2.Dispose(); } var producerConfig3 = new ProducerConfig(); producerConfig3.Brokers = new List <BrokerConfiguration> { new BrokerConfiguration { BrokerId = 0, Host = "localhost", Port = this.port1 }, new BrokerConfiguration { BrokerId = 1, Host = "localhost", Port = this.port2 } }; producerConfig3.KeySerializer = typeof(StringEncoder).AssemblyQualifiedName; producerConfig3.Serializer = typeof(StringEncoder).AssemblyQualifiedName; var producer3 = new Producer <string, string>(producerConfig2); try { producer3.Send(new KeyedMessage <string, string>(topic, "test", "test1")); } finally { producer3.Dispose(); } }