public void TestProduceAndMultiFetch()
{
// send some messages, with non-ordered topics
var topicOffsets = new List<Tuple<string, int>> { Tuple.Create("test4", 0), Tuple.Create("test1", 0), Tuple.Create("test2", 0), Tuple.Create("test3", 0) };
{
var messages = new Dictionary<string, List<string>>();
var builder = new FetchRequestBuilder();
foreach (var topicAndOffset in topicOffsets)
{
var topic = topicAndOffset.Item1;
var offset = topicAndOffset.Item2;
var producedData = new List<string> { "a_" + topic, "b_" + topic };
messages[topic] = producedData;
Producer.Send(producedData.Select(m => new KeyedMessage<string, string>(topic, topic, m)).ToArray());
TestUtils.WaitUntilMetadataIsPropagated(this.Servers, topic, 0, 1000);
builder.AddFetch(topic, offset, 0, 10000);
}
// wait a bit for produced message to be available
var request = builder.Build();
var response = Consumer.Fetch(request);
foreach (var topicAndOffset in topicOffsets)
{
var fetched = response.MessageSet(topicAndOffset.Item1, topicAndOffset.Item2);
Assert.Equal(messages[topicAndOffset.Item1], fetched.Select(m => Util.ReadString(m.Message.Payload)).ToList());
}
}
{
// send some invalid offsets
var builder = new FetchRequestBuilder();
foreach (var topicAndOffset in topicOffsets)
{
builder.AddFetch(topicAndOffset.Item1, topicAndOffset.Item2, -1, 10000);
}
var request = builder.Build();
var responses = Consumer.Fetch(request);
foreach (var pd in responses.Data.Values)
{
try
{
ErrorMapping.MaybeThrowException(pd.Error);
Assert.True(false, "Expected an OffsetOutOfRangeException exception to be thrown");
}
catch (OffsetOutOfRangeException)
{
// this is good
}
}
}
}
public void TestFetchRequestCanProperlySerialize()
{
var request =
new FetchRequestBuilder().ClientId("test-client")
.MaxWait(10001)
.MinBytes(4444)
.AddFetch("topic1", 0, 0, 10000)
.AddFetch("topic2", 1, 1024, 9999)
.AddFetch("topic1", 1, 256, 444)
.Build();
var serializedBuilder = ByteBuffer.Allocate(request.SizeInBytes);
request.WriteTo(serializedBuilder);
serializedBuilder.Rewind();
var deserializedRequest = FetchRequest.ReadFrom(serializedBuilder);
Assert.Equal(request, deserializedRequest);
}
public void TestDefaultEncoderProducerAndFetch()
{
var topic = "test-topic";
var config = Producer.Config;
var stringProducer1 = new Producer<string, string>(config);
stringProducer1.Send(new KeyedMessage<string, string>(topic, "test-message"));
// note we can't validate high watermark here
var request = new FetchRequestBuilder().ClientId("test-client").AddFetch(topic, 0, 0, 10000).Build();
var fetched = Consumer.Fetch(request);
Assert.Equal(0, fetched.CorrelationId);
var messageSet = fetched.MessageSet(topic, 0);
Assert.True(messageSet.Iterator().HasNext());
var fetchedMessageAndOffset = messageSet.Iterator().Next();
Assert.Equal("test-message", Util.ReadString(fetchedMessageAndOffset.Message.Payload));
}
internal AbstractFetcherThread(
string name,
string clientId,
Broker sourceBroker,
int socketTimeout,
int socketBufferSize,
int fetchSize,
int fetcherBrokerId = -1,
int maxWait = 0,
int minBytes = 1,
bool isInterruptible = true)
: base(name, isInterruptible)
{
this.clientId = clientId;
this.sourceBroker = sourceBroker;
this.socketTimeout = socketTimeout;
this.socketBufferSize = socketBufferSize;
this.fetchSize = fetchSize;
this.fetcherBrokerId = fetcherBrokerId;
this.maxWait = maxWait;
this.minBytes = minBytes;
this.partitionMapLock = new ReentrantLock();
this.partitionMapCond = this.partitionMapLock.NewCondition();
this.simpleConsumer = new SimpleConsumer(
sourceBroker.Host, sourceBroker.Port, socketTimeout, socketBufferSize, clientId);
this.brokerInfo = string.Format("host_{0}-port_{1}", sourceBroker.Host, sourceBroker.Port);
this.metricId = new ClientIdAndBroker(clientId, this.brokerInfo);
this.FetcherStats = new FetcherStats(this.metricId);
this.FetcherLagStats = new FetcherLagStats(this.metricId);
this.fetchRequestBuilder =
new FetchRequestBuilder().ClientId(clientId)
.ReplicaId(fetcherBrokerId)
.MaxWait(maxWait)
.MinBytes(minBytes);
}
public void TestPipelinedProduceRequests()
{
this.CreateSimpleTopicsAndAwaitLeader(this.ZkClient, new List<string> { "test1", "test2", "test3", "test4" }, Configs.First().BrokerId);
var props = Producer.Config;
props.RequestRequiredAcks = 0;
var pipelinedProducer = new Producer<string, string>(props);
// send some messages
var topics = new List<Tuple<string, int>>
{
Tuple.Create("test4", 0),
Tuple.Create("test1", 0),
Tuple.Create("test2", 0),
Tuple.Create("test3", 0)
};
var messages = new Dictionary<string, List<string>>();
var builder = new FetchRequestBuilder();
foreach (var topicAndPartition in topics)
{
var topic = topicAndPartition.Item1;
var partition = topicAndPartition.Item2;
var messageList = new List<string> { "a_" + topics, "b_" + topic };
var producerData = messageList.Select(m => new KeyedMessage<string, string>(topic, topic, m)).ToArray();
messages[topic] = messageList;
pipelinedProducer.Send(producerData);
builder.AddFetch(topic, partition, 0, 10000);
}
// wait until the messages are published
Thread.Sleep(7000); // ugly sleep as we can't access logManager endOffset
var request = builder.Build();
var response = Consumer.Fetch(request);
foreach (var topicAndPartition in topics)
{
var topic = topicAndPartition.Item1;
var partition = topicAndPartition.Item2;
var fetched = response.MessageSet(topic, partition);
Assert.Equal(messages[topic], fetched.Select(messageAndOffset => Util.ReadString(messageAndOffset.Message.Payload)).ToList());
}
}
public void TestMultiProduce()
{
this.CreateSimpleTopicsAndAwaitLeader(
this.ZkClient, new List<string> { "test1", "test2", "test3", "test4" }, Configs.First().BrokerId);
// send some messages
var topics = new List<Tuple<string, int>> { Tuple.Create("test4", 0), Tuple.Create("test1", 0), Tuple.Create("test2", 0), Tuple.Create("test3", 0) };
{
var messages = new Dictionary<string, List<string>>();
var builder = new FetchRequestBuilder();
foreach (var topicAndOffset in topics)
{
var topic = topicAndOffset.Item1;
var partition = topicAndOffset.Item2;
var messageList = new List<string> { "a_" + topic, "b_" + topic };
var producerData = messageList.Select(m => new KeyedMessage<string, string>(topic, topic, m)).ToArray();
messages[topic] = messageList;
Producer.Send(producerData);
builder.AddFetch(topic, partition, 0, 10000);
}
// wait a bit for produced message to be available
var request = builder.Build();
var response = Consumer.Fetch(request);
foreach (var topicAndPartition in topics)
{
var fetched = response.MessageSet(topicAndPartition.Item1, topicAndPartition.Item2);
Assert.Equal(messages[topicAndPartition.Item1], fetched.Select(m => Util.ReadString(m.Message.Payload)).ToList());
}
}
}
public void TestProduceAndMultiFetch()
{
this.CreateSimpleTopicsAndAwaitLeader(
this.ZkClient, new List<string> { "test1", "test2", "test3", "test4" }, Configs.First().BrokerId);
// send some messages, with non-ordered topics
var topics = new List<Tuple<string, int>> { Tuple.Create("test4", 0), Tuple.Create("test1", 0), Tuple.Create("test2", 0), Tuple.Create("test3", 0) };
{
var messages = new Dictionary<string, List<string>>();
var builder = new FetchRequestBuilder();
foreach (var topicAndOffset in topics)
{
var topic = topicAndOffset.Item1;
var partition = topicAndOffset.Item2;
var messageList = new List<string> { "a_" + topic, "b_" + topic };
var producerData = messageList.Select(m => new KeyedMessage<string, string>(topic, topic, m)).ToArray();
messages[topic] = messageList;
Producer.Send(producerData);
builder.AddFetch(topic, partition, 0, 10000);
}
// wait a bit for produced message to be available
var request = builder.Build();
var response = Consumer.Fetch(request);
foreach (var topicAndPartition in topics)
{
var fetched = response.MessageSet(topicAndPartition.Item1, topicAndPartition.Item2);
Assert.Equal(messages[topicAndPartition.Item1], fetched.Select(m => Util.ReadString(m.Message.Payload)).ToList());
}
}
{
// send some invalid offsets
var builder = new FetchRequestBuilder();
foreach (var topicAndPartition in topics)
{
builder.AddFetch(topicAndPartition.Item1, topicAndPartition.Item2, -1, 10000);
}
try
{
var request = builder.Build();
var response = Consumer.Fetch(request);
foreach (var pdata in response.Data.Values)
{
ErrorMapping.MaybeThrowException(pdata.Error);
}
Assert.True(false, "Expected exception when fetching message with invalid offset");
}
catch (OffsetOutOfRangeException)
{
// ok
}
}
{
// send some invalid partitions
var builder = new FetchRequestBuilder();
foreach (var topicAndPartition in topics)
{
builder.AddFetch(topicAndPartition.Item1, -1, 0, 10000);
}
try
{
var request = builder.Build();
var response = Consumer.Fetch(request);
foreach (var pdata in response.Data.Values)
{
ErrorMapping.MaybeThrowException(pdata.Error);
}
Assert.True(false, "Expected exception when fetching message with invalid partition");
}
catch (UnknownTopicOrPartitionException)
{
// ok
}
}
}
public void TestMultiProduceResend()
{
// send some messages
var topics = new List<string> { "test1", "test2", "test3" };
var messages = new Dictionary<string, List<string>>();
var builder = new FetchRequestBuilder();
var producerList = new List<KeyedMessage<string, string>>();
foreach (var topic in topics)
{
var set = new List<string> { "a_" + topic, "b_" + topic };
messages[topic] = set;
producerList.AddRange(set.Select(x => new KeyedMessage<string, string>(topic, topic, x)));
builder.AddFetch(topic, 0, 0, 10000);
}
Producer.Send(producerList.ToArray());
foreach (var topic in topics)
{
TestUtils.WaitUntilMetadataIsPropagated(this.Servers, topic, 0, 1000);
}
Producer.Send(producerList.ToArray());
// wait a bit for produced message to be available
var request = builder.Build();
var response = Consumer.Fetch(request);
foreach (var topic in topics)
{
var fetched = response.MessageSet(topic, 0);
var mergedList = new List<string>(messages[topic]);
mergedList.AddRange(messages[topic]);
Assert.Equal(mergedList, fetched.Select(m => Util.ReadString(m.Message.Payload)).ToList());
}
}