public void TestPipelinedProduceRequests()
{
this.CreateSimpleTopicsAndAwaitLeader(this.ZkClient, new List<string> { "test1", "test2", "test3", "test4" }, Configs.First().BrokerId);
var props = Producer.Config;
props.RequestRequiredAcks = 0;
var pipelinedProducer = new Producer<string, string>(props);
// send some messages
var topics = new List<Tuple<string, int>>
{
Tuple.Create("test4", 0),
Tuple.Create("test1", 0),
Tuple.Create("test2", 0),
Tuple.Create("test3", 0)
};
var messages = new Dictionary<string, List<string>>();
var builder = new FetchRequestBuilder();
foreach (var topicAndPartition in topics)
{
var topic = topicAndPartition.Item1;
var partition = topicAndPartition.Item2;
var messageList = new List<string> { "a_" + topics, "b_" + topic };
var producerData = messageList.Select(m => new KeyedMessage<string, string>(topic, topic, m)).ToArray();
messages[topic] = messageList;
pipelinedProducer.Send(producerData);
builder.AddFetch(topic, partition, 0, 10000);
}
// wait until the messages are published
Thread.Sleep(7000); // ugly sleep as we can't access logManager endOffset
var request = builder.Build();
var response = Consumer.Fetch(request);
foreach (var topicAndPartition in topics)
{
var topic = topicAndPartition.Item1;
var partition = topicAndPartition.Item2;
var fetched = response.MessageSet(topic, partition);
Assert.Equal(messages[topic], fetched.Select(messageAndOffset => Util.ReadString(messageAndOffset.Message.Payload)).ToList());
}
}
public void TestMultiProduce()
{
this.CreateSimpleTopicsAndAwaitLeader(
this.ZkClient, new List<string> { "test1", "test2", "test3", "test4" }, Configs.First().BrokerId);
// send some messages
var topics = new List<Tuple<string, int>> { Tuple.Create("test4", 0), Tuple.Create("test1", 0), Tuple.Create("test2", 0), Tuple.Create("test3", 0) };
{
var messages = new Dictionary<string, List<string>>();
var builder = new FetchRequestBuilder();
foreach (var topicAndOffset in topics)
{
var topic = topicAndOffset.Item1;
var partition = topicAndOffset.Item2;
var messageList = new List<string> { "a_" + topic, "b_" + topic };
var producerData = messageList.Select(m => new KeyedMessage<string, string>(topic, topic, m)).ToArray();
messages[topic] = messageList;
Producer.Send(producerData);
builder.AddFetch(topic, partition, 0, 10000);
}
// wait a bit for produced message to be available
var request = builder.Build();
var response = Consumer.Fetch(request);
foreach (var topicAndPartition in topics)
{
var fetched = response.MessageSet(topicAndPartition.Item1, topicAndPartition.Item2);
Assert.Equal(messages[topicAndPartition.Item1], fetched.Select(m => Util.ReadString(m.Message.Payload)).ToList());
}
}
}
public void TestProduceAndMultiFetch()
{
// send some messages, with non-ordered topics
var topicOffsets = new List<Tuple<string, int>> { Tuple.Create("test4", 0), Tuple.Create("test1", 0), Tuple.Create("test2", 0), Tuple.Create("test3", 0) };
{
var messages = new Dictionary<string, List<string>>();
var builder = new FetchRequestBuilder();
foreach (var topicAndOffset in topicOffsets)
{
var topic = topicAndOffset.Item1;
var offset = topicAndOffset.Item2;
var producedData = new List<string> { "a_" + topic, "b_" + topic };
messages[topic] = producedData;
Producer.Send(producedData.Select(m => new KeyedMessage<string, string>(topic, topic, m)).ToArray());
TestUtils.WaitUntilMetadataIsPropagated(this.Servers, topic, 0, 1000);
builder.AddFetch(topic, offset, 0, 10000);
}
// wait a bit for produced message to be available
var request = builder.Build();
var response = Consumer.Fetch(request);
foreach (var topicAndOffset in topicOffsets)
{
var fetched = response.MessageSet(topicAndOffset.Item1, topicAndOffset.Item2);
Assert.Equal(messages[topicAndOffset.Item1], fetched.Select(m => Util.ReadString(m.Message.Payload)).ToList());
}
}
{
// send some invalid offsets
var builder = new FetchRequestBuilder();
foreach (var topicAndOffset in topicOffsets)
{
builder.AddFetch(topicAndOffset.Item1, topicAndOffset.Item2, -1, 10000);
}
var request = builder.Build();
var responses = Consumer.Fetch(request);
foreach (var pd in responses.Data.Values)
{
try
{
ErrorMapping.MaybeThrowException(pd.Error);
Assert.True(false, "Expected an OffsetOutOfRangeException exception to be thrown");
}
catch (OffsetOutOfRangeException)
{
// this is good
}
}
}
}
public void TestProduceAndMultiFetch()
{
this.CreateSimpleTopicsAndAwaitLeader(
this.ZkClient, new List<string> { "test1", "test2", "test3", "test4" }, Configs.First().BrokerId);
// send some messages, with non-ordered topics
var topics = new List<Tuple<string, int>> { Tuple.Create("test4", 0), Tuple.Create("test1", 0), Tuple.Create("test2", 0), Tuple.Create("test3", 0) };
{
var messages = new Dictionary<string, List<string>>();
var builder = new FetchRequestBuilder();
foreach (var topicAndOffset in topics)
{
var topic = topicAndOffset.Item1;
var partition = topicAndOffset.Item2;
var messageList = new List<string> { "a_" + topic, "b_" + topic };
var producerData = messageList.Select(m => new KeyedMessage<string, string>(topic, topic, m)).ToArray();
messages[topic] = messageList;
Producer.Send(producerData);
builder.AddFetch(topic, partition, 0, 10000);
}
// wait a bit for produced message to be available
var request = builder.Build();
var response = Consumer.Fetch(request);
foreach (var topicAndPartition in topics)
{
var fetched = response.MessageSet(topicAndPartition.Item1, topicAndPartition.Item2);
Assert.Equal(messages[topicAndPartition.Item1], fetched.Select(m => Util.ReadString(m.Message.Payload)).ToList());
}
}
{
// send some invalid offsets
var builder = new FetchRequestBuilder();
foreach (var topicAndPartition in topics)
{
builder.AddFetch(topicAndPartition.Item1, topicAndPartition.Item2, -1, 10000);
}
try
{
var request = builder.Build();
var response = Consumer.Fetch(request);
foreach (var pdata in response.Data.Values)
{
ErrorMapping.MaybeThrowException(pdata.Error);
}
Assert.True(false, "Expected exception when fetching message with invalid offset");
}
catch (OffsetOutOfRangeException)
{
// ok
}
}
{
// send some invalid partitions
var builder = new FetchRequestBuilder();
foreach (var topicAndPartition in topics)
{
builder.AddFetch(topicAndPartition.Item1, -1, 0, 10000);
}
try
{
var request = builder.Build();
var response = Consumer.Fetch(request);
foreach (var pdata in response.Data.Values)
{
ErrorMapping.MaybeThrowException(pdata.Error);
}
Assert.True(false, "Expected exception when fetching message with invalid partition");
}
catch (UnknownTopicOrPartitionException)
{
// ok
}
}
}
public void TestMultiProduceResend()
{
// send some messages
var topics = new List<string> { "test1", "test2", "test3" };
var messages = new Dictionary<string, List<string>>();
var builder = new FetchRequestBuilder();
var producerList = new List<KeyedMessage<string, string>>();
foreach (var topic in topics)
{
var set = new List<string> { "a_" + topic, "b_" + topic };
messages[topic] = set;
producerList.AddRange(set.Select(x => new KeyedMessage<string, string>(topic, topic, x)));
builder.AddFetch(topic, 0, 0, 10000);
}
Producer.Send(producerList.ToArray());
foreach (var topic in topics)
{
TestUtils.WaitUntilMetadataIsPropagated(this.Servers, topic, 0, 1000);
}
Producer.Send(producerList.ToArray());
// wait a bit for produced message to be available
var request = builder.Build();
var response = Consumer.Fetch(request);
foreach (var topic in topics)
{
var fetched = response.MessageSet(topic, 0);
var mergedList = new List<string>(messages[topic]);
mergedList.AddRange(messages[topic]);
Assert.Equal(mergedList, fetched.Select(m => Util.ReadString(m.Message.Payload)).ToList());
}
}