public void TestMessageSizeTooLarge()
{
var props = TestUtils.GetSyncProducerConfig(this.Configs[0].Port);
var producer = new SyncProducer(props);
AdminUtils.CreateTopic(this.ZkClient, "test", 1, 1, new Dictionary <string, string>());
TestUtils.WaitUntilLeaderIsElectedOrChanged(this.ZkClient, "test", 0, 500);
TestUtils.WaitUntilMetadataIsPropagated(Servers, "test", 0, 2000);
var message1 = new Message(new byte[Configs[0].MessageMaxBytes + 1]);
var messageSet1 = new ByteBufferMessageSet(CompressionCodecs.NoCompressionCodec, new List <Message> {
message1
});
var response1 = producer.Send(TestUtils.ProduceRequest("test", 0, messageSet1, acks: 1));
Assert.Equal(1, response1.Status.Count(kvp => kvp.Value.Error != ErrorMapping.NoError));
Assert.Equal(ErrorMapping.MessageSizeTooLargeCode, response1.Status[new TopicAndPartition("test", 0)].Error);
Assert.Equal(-1L, response1.Status[new TopicAndPartition("test", 0)].Offset);
var safeSize = Configs[0].MessageMaxBytes - Message.MessageOverhead - MessageSet.LogOverhead - 1;
var message2 = new Message(new byte[safeSize]);
var messageSet2 = new ByteBufferMessageSet(
CompressionCodecs.NoCompressionCodec, new List <Message> {
message2
});
var response2 = producer.Send(TestUtils.ProduceRequest("test", 0, messageSet2, acks: 1));
Assert.Equal(0, response2.Status.Count(kvp => kvp.Value.Error != ErrorMapping.NoError));
Assert.Equal(ErrorMapping.NoError, response2.Status[new TopicAndPartition("test", 0)].Error);
Assert.Equal(0, response2.Status[new TopicAndPartition("test", 0)].Offset);
}
public void TestGetAllTopicMetadata()
{
// create topic
var topic1 = "testGetAllTopicMetadata1";
var topic2 = "testGetAllTopicMetadata2";
AdminUtils.CreateTopic(this.ZkClient, topic1, 1, 1, new Dictionary <string, string>());
AdminUtils.CreateTopic(this.ZkClient, topic2, 1, 1, new Dictionary <string, string>());
// wait for leader to be elected for both topics
TestUtils.WaitUntilMetadataIsPropagated(this.Servers, topic1, 0, 1000);
TestUtils.WaitUntilMetadataIsPropagated(this.Servers, topic2, 0, 1000);
// issue metadata request with empty list of topics
var topicsMetadata =
ClientUtils.FetchTopicMetadata(
new HashSet <string>(), this.Brokers, "TopicMetadataTest-testGetAllTopicMetadata", 2000).TopicsMetadata;
Assert.Equal(ErrorMapping.NoError, topicsMetadata.First().ErrorCode);
Assert.Equal(2, topicsMetadata.Count);
Assert.Equal(ErrorMapping.NoError, topicsMetadata.First().PartitionsMetadata.First().ErrorCode);
Assert.Equal(ErrorMapping.NoError, topicsMetadata.Last().PartitionsMetadata.First().ErrorCode);
var partitionMetadataTopic1 = topicsMetadata.First().PartitionsMetadata;
var partitionMetadataTopic2 = topicsMetadata.Last().PartitionsMetadata;
Assert.Equal(1, partitionMetadataTopic1.Count);
Assert.Equal(0, partitionMetadataTopic1.First().PartitionId);
Assert.Equal(1, partitionMetadataTopic1.First().Replicas.Count());
Assert.Equal(1, partitionMetadataTopic2.Count);
Assert.Equal(0, partitionMetadataTopic2.First().PartitionId);
Assert.Equal(1, partitionMetadataTopic2.First().Replicas.Count());
}
public void TestSendNullMessage()
{
var config = new ProducerConfig
{
Serializer = typeof(StringEncoder).AssemblyQualifiedName,
KeySerializer = typeof(StringEncoder).AssemblyQualifiedName,
PartitionerClass = typeof(StaticPartitioner).AssemblyQualifiedName,
Brokers =
TestUtils.GetBrokerListFromConfigs(
new List <TempKafkaConfig> {
this.config1, this.config2
}),
};
var producer = new Producer <string, string>(config);
try
{
// create topic
AdminUtils.CreateTopic(this.ZkClient, "new-topic", 2, 1, new Dictionary <string, string>());
TestUtils.WaitUntilLeaderIsElectedOrChanged(this.ZkClient, "new-topic", 0, 500);
producer.Send(new KeyedMessage <string, string>("new-topic", "key", null));
}
finally
{
producer.Dispose();
}
}
public void CreateSimpleTopicsAndAwaitLeader(ZkClient zkClient, List <string> topics, int brokerId)
{
foreach (var topic in topics)
{
AdminUtils.CreateTopic(zkClient, topic, 1, 1, new Dictionary <string, string>());
TestUtils.WaitUntilLeaderIsElectedOrChanged(this.ZkClient, topic, 0, 500);
}
}
public void TestConsumerEmptyTopic()
{
var newTopic = "new-topic";
AdminUtils.CreateTopic(this.ZkClient, newTopic, 1, 1, new Dictionary <string, string>());
TestUtils.WaitUntilMetadataIsPropagated(this.Servers, newTopic, 0, 1000);
TestUtils.WaitUntilLeaderIsElectedOrChanged(this.ZkClient, newTopic, 0, 500);
var fetchResponse = Consumer.Fetch(new FetchRequestBuilder().AddFetch(newTopic, 0, 0, 10000).Build());
Assert.False(fetchResponse.MessageSet(newTopic, 0).Iterator().HasNext());
}
public void TestProduceCorrectlyReceivesResponse()
{
var props = TestUtils.GetSyncProducerConfig(this.Configs[0].Port);
var producer = new SyncProducer(props);
var messages = new ByteBufferMessageSet(
CompressionCodecs.NoCompressionCodec, new List <Message> {
new Message(this.messageBytes)
});
// #1 - test that we get an error when partition does not belong to broker in response
var request = TestUtils.ProduceRequestWithAcks(
new List <string> {
"topic1", "topic2", "topic3"
}, new List <int> {
0
}, messages, 1);
var response = producer.Send(request);
Assert.NotNull(response);
Assert.Equal(request.CorrelationId, response.CorrelationId);
Assert.Equal(3, response.Status.Count);
foreach (var responseStatus in response.Status.Values)
{
Assert.Equal(ErrorMapping.UnknownTopicOrPartitionCode, responseStatus.Error);
Assert.Equal(-1L, responseStatus.Offset);
}
// #2 - test that we get correct offsets when partition is owned by broker
AdminUtils.CreateTopic(this.ZkClient, "topic1", 1, 1, new Dictionary <string, string>());
AdminUtils.CreateTopic(this.ZkClient, "topic3", 1, 1, new Dictionary <string, string>());
TestUtils.WaitUntilLeaderIsElectedOrChanged(this.ZkClient, "topic3", 0, 5000);
TestUtils.WaitUntilMetadataIsPropagated(Servers, "topic3", 0, 2000);
var response2 = producer.Send(request);
Assert.NotNull(response2);
Assert.Equal(request.CorrelationId, response2.CorrelationId);
Assert.Equal(3, response2.Status.Count);
// the first and last message should have been accepted by broker
Assert.Equal(ErrorMapping.NoError, response2.Status[new TopicAndPartition("topic1", 0)].Error);
Assert.Equal(ErrorMapping.NoError, response2.Status[new TopicAndPartition("topic3", 0)].Error);
Assert.Equal(0, response2.Status[new TopicAndPartition("topic1", 0)].Offset);
Assert.Equal(0, response2.Status[new TopicAndPartition("topic3", 0)].Offset);
// the middle message should have been rejected because broker doesn't lead partition
Assert.Equal(ErrorMapping.UnknownTopicOrPartitionCode, response2.Status[new TopicAndPartition("topic2", 0)].Error);
Assert.Equal(-1L, response2.Status[new TopicAndPartition("topic2", 0)].Offset);
}
public void TestLeaderSelectionForPartition()
{
var zkClient = new ZkClient(this.zookeeperConnect, 6000, 30000, new ZkStringSerializer());
// create topic topic1 with 1 partition on broker 0
AdminUtils.CreateTopic(zkClient, Topic, 1, 1, new Dictionary <string, string>());
TestUtils.WaitUntilMetadataIsPropagated(this.Servers, Topic, 0, 3000);
var sentMessages1 = this.SendMessages(
Configs.First(), nMessages, "batch1", CompressionCodecs.NoCompressionCodec, 1);
TestUtils.WaitUntilMetadataIsPropagated(this.Servers, Topic, 0, 1000);
// create a consuemr
var consumerConfig1 = TestUtils.CreateConsumerProperties(ZkConnect, Group, Consumer1);
var zkConsumerConnector1 = new ZookeeperConsumerConnector(consumerConfig1);
var topicMessageStreams1 =
zkConsumerConnector1.CreateMessageStreams(
new Dictionary <string, int> {
{ Topic, 1 }
}, new StringDecoder(), new StringDecoder());
var topicRegistry = zkConsumerConnector1.TopicRegistry;
Assert.Equal(1, topicRegistry.Select(x => x.Key).Count());
Assert.Equal(Topic, topicRegistry.Select(x => x.Key).First());
var topicsAndPartitionsInRegistry =
topicRegistry.Select(x => Tuple.Create(x.Key, x.Value.Select(p => p.Value))).ToList();
var brokerPartition = topicsAndPartitionsInRegistry.First().Item2.First();
Assert.Equal(0, brokerPartition.PartitionId);
// also check partition ownership
var actual_1 = this.GetZKChildrenValues(this.dirs.ConsumerOwnerDir);
var expected_1 = new List <Tuple <string, string> >
{
Tuple.Create("0", "group1_consumer1-0"),
};
Assert.Equal(expected_1, actual_1);
var receivedMessages1 = this.GetMessages(nMessages, topicMessageStreams1);
Assert.Equal(sentMessages1, receivedMessages1);
zkConsumerConnector1.Shutdown();
zkClient.Dispose();
}
public void TestTopicMetadataRequest()
{
// create topic
var topic = "test";
AdminUtils.CreateTopic(this.ZkClient, topic, 1, 1, new Dictionary <string, string>());
// create a topic metadata request
var topicMetadataRequest = new TopicMetadataRequest(new List <string> {
topic
}, 0);
var serializedMetadataRequest = ByteBuffer.Allocate(topicMetadataRequest.SizeInBytes + 2);
topicMetadataRequest.WriteTo(serializedMetadataRequest);
serializedMetadataRequest.Rewind();
var deserializedMetadataRequest = TopicMetadataRequest.ReadFrom(serializedMetadataRequest);
Assert.Equal(topicMetadataRequest, deserializedMetadataRequest);
}
public void TestMessageSizeTooLargeWithAckZero()
{
var props = TestUtils.GetSyncProducerConfig(this.Configs[0].Port);
props.RequestRequiredAcks = 0;
var producer = new SyncProducer(props);
AdminUtils.CreateTopic(this.ZkClient, "test", 1, 1, new Dictionary <string, string>());
TestUtils.WaitUntilLeaderIsElectedOrChanged(this.ZkClient, "test", 0, 500);
// This message will be dropped silently since message size too large.
producer.Send(
TestUtils.ProduceRequest(
"test",
0,
new ByteBufferMessageSet(
CompressionCodecs.NoCompressionCodec,
new List <Message> {
new Message(new byte[Configs[0].MessageMaxBytes + 1])
})));
// Send another message whose size is large enough to exceed the buffer size so
// the socket buffer will be flushed immediately;
// this send should fail since the socket has been closed
try
{
producer.Send(
TestUtils.ProduceRequest(
"test",
0,
new ByteBufferMessageSet(
CompressionCodecs.NoCompressionCodec,
new List <Message> {
new Message(new byte[Configs[0].MessageMaxBytes + 1])
})));
}
catch (IOException)
{
}
}
public void TestBasicTopicMetadata()
{
// create topic
var topic = "test";
AdminUtils.CreateTopic(this.ZkClient, topic, 1, 1, new Dictionary <string, string>());
TestUtils.WaitUntilMetadataIsPropagated(this.Servers, topic, 0, 1000);
TestUtils.WaitUntilLeaderIsElectedOrChanged(this.ZkClient, topic, 0, 1000);
var topicsMetadata = ClientUtils.FetchTopicMetadata(
new HashSet <string> {
topic
}, this.Brokers, "TopicMetadataTest-testBasicTopicMetadata", 2000)
.TopicsMetadata;
Assert.Equal(ErrorMapping.NoError, topicsMetadata.First().ErrorCode);
Assert.Equal(ErrorMapping.NoError, topicsMetadata.First().PartitionsMetadata.First().ErrorCode);
Assert.Equal(1, topicsMetadata.Count);
Assert.Equal("test", topicsMetadata.First().Topic);
var partitionMetadata = topicsMetadata.First().PartitionsMetadata;
Assert.Equal(1, partitionMetadata.Count);
Assert.Equal(0, partitionMetadata.First().PartitionId);
Assert.Equal(1, partitionMetadata.First().Replicas.Count());
}
public void TestSendToNewTopic()
{
var producerConfig1 = new ProducerConfig
{
Serializer = typeof(StringEncoder).AssemblyQualifiedName,
KeySerializer = typeof(StringEncoder).AssemblyQualifiedName,
PartitionerClass =
typeof(StaticPartitioner).AssemblyQualifiedName,
Brokers =
TestUtils.GetBrokerListFromConfigs(
new List <TempKafkaConfig> {
this.config1, this.config2
}),
RequestRequiredAcks = 2,
RequestTimeoutMs = 1000
};
var producerConfig2 = new ProducerConfig
{
Serializer = typeof(StringEncoder).AssemblyQualifiedName,
KeySerializer = typeof(StringEncoder).AssemblyQualifiedName,
PartitionerClass =
typeof(StaticPartitioner).AssemblyQualifiedName,
Brokers =
TestUtils.GetBrokerListFromConfigs(
new List <TempKafkaConfig> {
this.config1, this.config2
}),
RequestRequiredAcks = 3,
RequestTimeoutMs = 1000
};
var topic = "new-topic";
// create topic with 1 partition and await leadership
AdminUtils.CreateTopic(this.ZkClient, topic, 1, 2, new Dictionary <string, string>());
TestUtils.WaitUntilMetadataIsPropagated(this.servers, topic, 0, 1000);
TestUtils.WaitUntilLeaderIsElectedOrChanged(this.ZkClient, topic, 0, 500);
var producer1 = new Producer <string, string>(producerConfig1);
var producer2 = new Producer <string, string>(producerConfig2);
// Available partition ids should be 0.
producer1.Send(new KeyedMessage <string, string>(topic, "test", "test1"));
producer1.Send(new KeyedMessage <string, string>(topic, "test", "test2"));
// get the leader
var leaderOpt = ZkUtils.GetLeaderForPartition(ZkClient, topic, 0);
Assert.True(leaderOpt.HasValue);
var leader = leaderOpt.Value;
var messageSet = (leader == this.config1.BrokerId)
? this.consumer1.Fetch(new FetchRequestBuilder().AddFetch(topic, 0, 0, 10000).Build())
.MessageSet("new-topic", 0)
.Iterator()
.ToEnumerable()
.ToList()
: this.consumer2.Fetch(new FetchRequestBuilder().AddFetch(topic, 0, 0, 10000).Build())
.MessageSet("new-topic", 0)
.Iterator()
.ToEnumerable()
.ToList();
Assert.Equal(2, messageSet.Count());
Assert.Equal(new Message(Encoding.UTF8.GetBytes("test1"), Encoding.UTF8.GetBytes("test")), messageSet[0].Message);
Assert.Equal(new Message(Encoding.UTF8.GetBytes("test2"), Encoding.UTF8.GetBytes("test")), messageSet[1].Message);
producer1.Dispose();
try
{
producer2.Send(new KeyedMessage <string, string>(topic, "test", "test2"));
Assert.False(true, "Should have timed out for 3 acks.");
}
catch (FailedToSendMessageException)
{
}
finally
{
producer2.Dispose();
}
}
public void TestUpdateBrokerPartitionInfo()
{
var topic = "new-topic";
AdminUtils.CreateTopic(this.ZkClient, topic, 1, 2, new Dictionary <string, string>());
// wait until the update metadata request for new topic reaches all servers
TestUtils.WaitUntilMetadataIsPropagated(this.servers, topic, 0, 3000);
TestUtils.WaitUntilLeaderIsElectedOrChanged(this.ZkClient, topic, 0, 500);
var producerConfig1 = new ProducerConfig();
producerConfig1.Brokers = new List <BrokerConfiguration>
{
new BrokerConfiguration
{
BrokerId = 0,
Host = "localhost",
Port = 18203
},
new BrokerConfiguration
{
BrokerId = 1,
Host = "localhost",
Port = 18204
}
};
producerConfig1.KeySerializer = typeof(StringEncoder).AssemblyQualifiedName;
producerConfig1.Serializer = typeof(StringEncoder).AssemblyQualifiedName;
var producer1 = new Producer <string, string>(producerConfig1);
try
{
producer1.Send(new KeyedMessage <string, string>(topic, "test", "test1"));
Assert.False(true, "Test should fail because the broker list provided are not valid");
}
catch (FailedToSendMessageException)
{
// ok
}
finally
{
producer1.Dispose();
}
var producerConfig2 = new ProducerConfig();
producerConfig2.Brokers = new List <BrokerConfiguration>
{
new BrokerConfiguration
{
BrokerId = 0,
Host = "localhost",
Port = 18203
},
new BrokerConfiguration
{
BrokerId = 1,
Host = "localhost",
Port = this.port1
}
};
producerConfig2.KeySerializer = typeof(StringEncoder).AssemblyQualifiedName;
producerConfig2.Serializer = typeof(StringEncoder).AssemblyQualifiedName;
var producer2 = new Producer <string, string>(producerConfig2);
try
{
producer2.Send(new KeyedMessage <string, string>(topic, "test", "test1"));
}
finally
{
producer2.Dispose();
}
var producerConfig3 = new ProducerConfig();
producerConfig3.Brokers = new List <BrokerConfiguration>
{
new BrokerConfiguration
{
BrokerId = 0,
Host = "localhost",
Port = this.port1
},
new BrokerConfiguration
{
BrokerId = 1,
Host = "localhost",
Port = this.port2
}
};
producerConfig3.KeySerializer = typeof(StringEncoder).AssemblyQualifiedName;
producerConfig3.Serializer = typeof(StringEncoder).AssemblyQualifiedName;
var producer3 = new Producer <string, string>(producerConfig2);
try
{
producer3.Send(new KeyedMessage <string, string>(topic, "test", "test1"));
}
finally
{
producer3.Dispose();
}
}
