public void TestDefaultEncoderProducerAndFetchWithCompression()
{
var topic = "test-topic";
var config = Producer.Config;
config.CompressionCodec = CompressionCodecs.DefaultCompressionCodec;
config.KeySerializer = typeof(StringEncoder).AssemblyQualifiedName;
config.Serializer = typeof(StringEncoder).AssemblyQualifiedName;
var stringProducer1 = new Producer <string, string>(config);
stringProducer1.Send(new KeyedMessage <string, string>(topic, "test-message"));
// note we can't validate high watermark here
var request = new FetchRequestBuilder().ClientId("test-client").AddFetch(topic, 0, 0, 10000).Build();
var fetched = Consumer.Fetch(request);
Assert.Equal(0, fetched.CorrelationId);
var messageSet = fetched.MessageSet(topic, 0);
Assert.True(messageSet.Iterator().HasNext());
var fetchedMessageAndOffset = messageSet.Iterator().Next();
Assert.Equal("test-message", Util.ReadString(fetchedMessageAndOffset.Message.Payload));
}
public void FetchRequestBuilderCustom()
{
var builder = new FetchRequestBuilder();
var topic = "topic";
var partitionId = 1;
long offset = 2;
var fetchSize = 3;
var correlationId = 4;
var clientId = "myClient";
var maxWait = 400;
var minBytes = 500;
var request =
builder.AddFetch(topic, partitionId, offset, fetchSize).CorrelationId(correlationId).ClientId(clientId).
MaxWait(maxWait).MinBytes(minBytes).Build();
Assert.IsNotNull(request);
Assert.AreEqual(FetchRequest.CurrentVersion, request.VersionId);
Assert.AreEqual(correlationId, request.CorrelationId);
Assert.AreEqual(clientId, request.ClientId);
Assert.AreEqual(maxWait, request.MaxWait);
Assert.AreEqual(minBytes, request.MinBytes);
Assert.AreEqual(1, request.OffsetInfo.Count());
Assert.AreEqual(topic, request.OffsetInfo.First().Key);
Assert.AreEqual(partitionId, request.OffsetInfo.First().Value.ToArray()[0].PartitionId);
Assert.AreEqual(offset, request.OffsetInfo.First().Value.ToArray()[0].Offset);
Assert.AreEqual(fetchSize, request.OffsetInfo.First().Value.ToArray()[0].FetchSize);
}
public void FetchRequestBuilderCustom()
{
var builder = new FetchRequestBuilder();
var topic = "topic";
var partitionId = 1;
long offset = 2;
var fetchSize = 3;
var correlationId = 4;
var clientId = "myClient";
var maxWait = 400;
var minBytes = 500;
var request =
builder.AddFetch(topic, partitionId, offset, fetchSize).CorrelationId(correlationId).ClientId(clientId).
MaxWait(maxWait).MinBytes(minBytes).Build();
Assert.IsNotNull(request);
Assert.AreEqual(FetchRequest.CurrentVersion, request.VersionId);
Assert.AreEqual(correlationId, request.CorrelationId);
Assert.AreEqual(clientId, request.ClientId);
Assert.AreEqual(maxWait, request.MaxWait);
Assert.AreEqual(minBytes, request.MinBytes);
Assert.AreEqual(1, request.OffsetInfo.Count());
Assert.AreEqual(topic, request.OffsetInfo.First().Key);
Assert.AreEqual(partitionId, request.OffsetInfo.First().Value.ToArray()[0].PartitionId);
Assert.AreEqual(offset, request.OffsetInfo.First().Value.ToArray()[0].Offset);
Assert.AreEqual(fetchSize, request.OffsetInfo.First().Value.ToArray()[0].FetchSize);
}
public void TestProduceAndMultiFetch()
{
// send some messages, with non-ordered topics
var topicOffsets = new List <Tuple <string, int> > {
Tuple.Create("test4", 0), Tuple.Create("test1", 0), Tuple.Create("test2", 0), Tuple.Create("test3", 0)
};
{
var messages = new Dictionary <string, List <string> >();
var builder = new FetchRequestBuilder();
foreach (var topicAndOffset in topicOffsets)
{
var topic = topicAndOffset.Item1;
var offset = topicAndOffset.Item2;
var producedData = new List <string> {
"a_" + topic, "b_" + topic
};
messages[topic] = producedData;
Producer.Send(producedData.Select(m => new KeyedMessage <string, string>(topic, topic, m)).ToArray());
TestUtils.WaitUntilMetadataIsPropagated(this.Servers, topic, 0, 1000);
builder.AddFetch(topic, offset, 0, 10000);
}
// wait a bit for produced message to be available
var request = builder.Build();
var response = Consumer.Fetch(request);
foreach (var topicAndOffset in topicOffsets)
{
var fetched = response.MessageSet(topicAndOffset.Item1, topicAndOffset.Item2);
Assert.Equal(messages[topicAndOffset.Item1], fetched.Select(m => Util.ReadString(m.Message.Payload)).ToList());
}
}
{
// send some invalid offsets
var builder = new FetchRequestBuilder();
foreach (var topicAndOffset in topicOffsets)
{
builder.AddFetch(topicAndOffset.Item1, topicAndOffset.Item2, -1, 10000);
}
var request = builder.Build();
var responses = Consumer.Fetch(request);
foreach (var pd in responses.Data.Values)
{
try
{
ErrorMapping.MaybeThrowException(pd.Error);
Assert.True(false, "Expected an OffsetOutOfRangeException exception to be thrown");
}
catch (OffsetOutOfRangeException)
{
// this is good
}
}
}
}
public void TestPipelinedProduceRequests()
{
this.CreateSimpleTopicsAndAwaitLeader(this.ZkClient, new List <string> {
"test1", "test2", "test3", "test4"
}, Configs.First().BrokerId);
var props = Producer.Config;
props.RequestRequiredAcks = 0;
var pipelinedProducer = new Producer <string, string>(props);
// send some messages
var topics = new List <Tuple <string, int> >
{
Tuple.Create("test4", 0),
Tuple.Create("test1", 0),
Tuple.Create("test2", 0),
Tuple.Create("test3", 0)
};
var messages = new Dictionary <string, List <string> >();
var builder = new FetchRequestBuilder();
foreach (var topicAndPartition in topics)
{
var topic = topicAndPartition.Item1;
var partition = topicAndPartition.Item2;
var messageList = new List <string> {
"a_" + topics, "b_" + topic
};
var producerData = messageList.Select(m => new KeyedMessage <string, string>(topic, topic, m)).ToArray();
messages[topic] = messageList;
pipelinedProducer.Send(producerData);
builder.AddFetch(topic, partition, 0, 10000);
}
// wait until the messages are published
Thread.Sleep(7000); // ugly sleep as we can't access logManager endOffset
var request = builder.Build();
var response = Consumer.Fetch(request);
foreach (var topicAndPartition in topics)
{
var topic = topicAndPartition.Item1;
var partition = topicAndPartition.Item2;
var fetched = response.MessageSet(topic, partition);
Assert.Equal(messages[topic], fetched.Select(messageAndOffset => Util.ReadString(messageAndOffset.Message.Payload)).ToList());
}
}
public void TestFetchRequestCanProperlySerialize()
{
var request =
new FetchRequestBuilder().ClientId("test-client")
.MaxWait(10001)
.MinBytes(4444)
.AddFetch("topic1", 0, 0, 10000)
.AddFetch("topic2", 1, 1024, 9999)
.AddFetch("topic1", 1, 256, 444)
.Build();
var serializedBuilder = ByteBuffer.Allocate(request.SizeInBytes);
request.WriteTo(serializedBuilder);
serializedBuilder.Rewind();
var deserializedRequest = FetchRequest.ReadFrom(serializedBuilder);
Assert.Equal(request, deserializedRequest);
}
public void TestMultiProduceResend()
{
// send some messages
var topics = new List <string> {
"test1", "test2", "test3"
};
var messages = new Dictionary <string, List <string> >();
var builder = new FetchRequestBuilder();
var producerList = new List <KeyedMessage <string, string> >();
foreach (var topic in topics)
{
var set = new List <string> {
"a_" + topic, "b_" + topic
};
messages[topic] = set;
producerList.AddRange(set.Select(x => new KeyedMessage <string, string>(topic, topic, x)));
builder.AddFetch(topic, 0, 0, 10000);
}
Producer.Send(producerList.ToArray());
foreach (var topic in topics)
{
TestUtils.WaitUntilMetadataIsPropagated(this.Servers, topic, 0, 1000);
}
Producer.Send(producerList.ToArray());
// wait a bit for produced message to be available
var request = builder.Build();
var response = Consumer.Fetch(request);
foreach (var topic in topics)
{
var fetched = response.MessageSet(topic, 0);
var mergedList = new List <string>(messages[topic]);
mergedList.AddRange(messages[topic]);
Assert.Equal(mergedList, fetched.Select(m => Util.ReadString(m.Message.Payload)).ToList());
}
}
public void TestMultiProduce()
{
this.CreateSimpleTopicsAndAwaitLeader(
this.ZkClient, new List <string> {
"test1", "test2", "test3", "test4"
}, Configs.First().BrokerId);
// send some messages
var topics = new List <Tuple <string, int> > {
Tuple.Create("test4", 0), Tuple.Create("test1", 0), Tuple.Create("test2", 0), Tuple.Create("test3", 0)
};
{
var messages = new Dictionary <string, List <string> >();
var builder = new FetchRequestBuilder();
foreach (var topicAndOffset in topics)
{
var topic = topicAndOffset.Item1;
var partition = topicAndOffset.Item2;
var messageList = new List <string> {
"a_" + topic, "b_" + topic
};
var producerData = messageList.Select(m => new KeyedMessage <string, string>(topic, topic, m)).ToArray();
messages[topic] = messageList;
Producer.Send(producerData);
builder.AddFetch(topic, partition, 0, 10000);
}
// wait a bit for produced message to be available
var request = builder.Build();
var response = Consumer.Fetch(request);
foreach (var topicAndPartition in topics)
{
var fetched = response.MessageSet(topicAndPartition.Item1, topicAndPartition.Item2);
Assert.Equal(messages[topicAndPartition.Item1], fetched.Select(m => Util.ReadString(m.Message.Payload)).ToList());
}
}
}
internal AbstractFetcherThread(
string name,
string clientId,
Broker sourceBroker,
int socketTimeout,
int socketBufferSize,
int fetchSize,
int fetcherBrokerId = -1,
int maxWait = 0,
int minBytes = 1,
bool isInterruptible = true)
: base(name, isInterruptible)
{
this.clientId = clientId;
this.sourceBroker = sourceBroker;
this.socketTimeout = socketTimeout;
this.socketBufferSize = socketBufferSize;
this.fetchSize = fetchSize;
this.fetcherBrokerId = fetcherBrokerId;
this.maxWait = maxWait;
this.minBytes = minBytes;
this.partitionMapLock = new ReentrantLock();
this.partitionMapCond = this.partitionMapLock.NewCondition();
this.simpleConsumer = new SimpleConsumer(
sourceBroker.Host, sourceBroker.Port, socketTimeout, socketBufferSize, clientId);
this.brokerInfo = string.Format("host_{0}-port_{1}", sourceBroker.Host, sourceBroker.Port);
this.metricId = new ClientIdAndBroker(clientId, this.brokerInfo);
this.FetcherStats = new FetcherStats(this.metricId);
this.FetcherLagStats = new FetcherLagStats(this.metricId);
this.fetchRequestBuilder =
new FetchRequestBuilder().ClientId(clientId)
.ReplicaId(fetcherBrokerId)
.MaxWait(maxWait)
.MinBytes(minBytes);
}
public void FetchRequestBuilderMostlyDefault()
{
var builder = new FetchRequestBuilder();
var topic = "topic";
var partitionId = 1;
long offset = 2;
var fetchSize = 3;
var request = builder.AddFetch(topic, partitionId, offset, fetchSize).Build();
Assert.IsNotNull(request);
Assert.AreEqual(FetchRequest.CurrentVersion, request.VersionId);
Assert.AreEqual(-1, request.CorrelationId);
Assert.AreEqual(string.Empty, request.ClientId);
Assert.AreEqual(-1, request.ReplicaId);
Assert.AreEqual(-1, request.MaxWait);
Assert.AreEqual(-1, request.MinBytes);
Assert.AreEqual(1, request.OffsetInfo.Count());
Assert.AreEqual(topic, request.OffsetInfo.First().Key);
Assert.AreEqual(partitionId, request.OffsetInfo.First().Value.ToArray()[0].PartitionId);
Assert.AreEqual(offset, request.OffsetInfo.First().Value.ToArray()[0].Offset);
Assert.AreEqual(fetchSize, request.OffsetInfo.First().Value.ToArray()[0].FetchSize);
}
public void FetchRequestBuilderMostlyDefault()
{
var builder = new FetchRequestBuilder();
var topic = "topic";
var partitionId = 1;
long offset = 2;
var fetchSize = 3;
var request = builder.AddFetch(topic, partitionId, offset, fetchSize).Build();
Assert.IsNotNull(request);
Assert.AreEqual(FetchRequest.CurrentVersion, request.VersionId);
Assert.AreEqual(-1, request.CorrelationId);
Assert.AreEqual(string.Empty, request.ClientId);
Assert.AreEqual(-1, request.ReplicaId);
Assert.AreEqual(-1, request.MaxWait);
Assert.AreEqual(-1, request.MinBytes);
Assert.AreEqual(1, request.OffsetInfo.Count());
Assert.AreEqual(topic, request.OffsetInfo.First().Key);
Assert.AreEqual(partitionId, request.OffsetInfo.First().Value.ToArray()[0].PartitionId);
Assert.AreEqual(offset, request.OffsetInfo.First().Value.ToArray()[0].Offset);
Assert.AreEqual(fetchSize, request.OffsetInfo.First().Value.ToArray()[0].FetchSize);
}
/// <summary>
/// Method to be used for starting a new thread
/// </summary>
internal void Run()
{
foreach (PartitionTopicInfo partitionTopicInfo in _partitionTopicInfos)
{
Logger.InfoFormat("{0} start fetching topic: {1} part: {2} offset: {3} from {4}:{5}",
_name,
partitionTopicInfo.Topic,
partitionTopicInfo.PartitionId,
partitionTopicInfo.NextRequestOffset,
_broker.Host,
_broker.Port);
}
int reqId = 0;
while (!_shouldStop && _partitionTopicInfos.Any())
{
try
{
IEnumerable <PartitionTopicInfo> fetchablePartitionTopicInfos = _partitionTopicInfos.Where(pti => pti.NextRequestOffset - pti.ConsumeOffset < _fetchBufferLength);
long read = 0;
if (fetchablePartitionTopicInfos.Any())
{
FetchRequestBuilder builder =
new FetchRequestBuilder().
CorrelationId(reqId).
ClientId(_config.ConsumerId ?? _name).
MaxWait(0).
MinBytes(0);
fetchablePartitionTopicInfos.ForEach(pti => builder.AddFetch(pti.Topic, pti.PartitionId, pti.NextRequestOffset, _config.FetchSize));
FetchRequest fetchRequest = builder.Build();
Logger.Debug("Sending fetch request: " + fetchRequest);
FetchResponse response = _simpleConsumer.Fetch(fetchRequest);
Logger.Debug("Fetch request completed");
var partitonsWithErrors = new List <PartitionTopicInfo>();
foreach (PartitionTopicInfo partitionTopicInfo in fetchablePartitionTopicInfos)
{
BufferedMessageSet messages = response.MessageSet(partitionTopicInfo.Topic, partitionTopicInfo.PartitionId);
switch (messages.ErrorCode)
{
case (short)ErrorMapping.NoError:
int bytesRead = partitionTopicInfo.Add(messages);
// TODO: The highwater offset on the message set is the end of the log partition. If the message retrieved is -1 of that offset, we are at the end.
if (messages.Messages.Any())
{
partitionTopicInfo.NextRequestOffset = messages.Messages.Last().Offset + 1;
read += bytesRead;
}
else
{
Logger.DebugFormat("No message returned by FetchRequest: {0}", fetchRequest.ToString());
}
break;
case (short)ErrorMapping.OffsetOutOfRangeCode:
try
{
Logger.InfoFormat("offset for {0} out of range", partitionTopicInfo);
long resetOffset = ResetConsumerOffsets(partitionTopicInfo.Topic,
partitionTopicInfo.PartitionId);
if (resetOffset >= 0)
{
partitionTopicInfo.FetchOffset = resetOffset;
partitionTopicInfo.ConsumeOffset = resetOffset;
Logger.InfoFormat("{0} marked as done.", partitionTopicInfo);
}
}
catch (Exception ex)
{
Logger.ErrorFormat("Error getting offsets for partition {0} : {1}", partitionTopicInfo.PartitionId, ex.FormatException());
partitonsWithErrors.Add(partitionTopicInfo);
}
break;
default:
Logger.ErrorFormat("Error returned from broker {2} for partition {0} : KafkaErrorCode: {1}", partitionTopicInfo.PartitionId, messages.ErrorCode, partitionTopicInfo.BrokerId);
partitonsWithErrors.Add(partitionTopicInfo);
break;
}
}
reqId = reqId == int.MaxValue ? 0 : reqId + 1;
if (partitonsWithErrors.Any())
{
RemovePartitionsFromProessing(partitonsWithErrors);
}
}
if (read > 0)
{
Logger.Debug("Fetched bytes: " + read);
}
if (read == 0)
{
Logger.DebugFormat("backing off {0} ms", _config.BackOffIncrement);
Thread.Sleep(_config.BackOffIncrement);
}
}
catch (Exception ex)
{
if (_shouldStop)
{
Logger.InfoFormat("FetcherRunnable {0} interrupted", this);
}
else
{
Logger.ErrorFormat("error in FetcherRunnable {0}", ex.FormatException());
}
}
}
Logger.InfoFormat("stopping fetcher {0} to host {1}", _name, _broker.Host);
}
public void TestProduceAndMultiFetch()
{
this.CreateSimpleTopicsAndAwaitLeader(
this.ZkClient, new List <string> {
"test1", "test2", "test3", "test4"
}, Configs.First().BrokerId);
// send some messages, with non-ordered topics
var topics = new List <Tuple <string, int> > {
Tuple.Create("test4", 0), Tuple.Create("test1", 0), Tuple.Create("test2", 0), Tuple.Create("test3", 0)
};
{
var messages = new Dictionary <string, List <string> >();
var builder = new FetchRequestBuilder();
foreach (var topicAndOffset in topics)
{
var topic = topicAndOffset.Item1;
var partition = topicAndOffset.Item2;
var messageList = new List <string> {
"a_" + topic, "b_" + topic
};
var producerData = messageList.Select(m => new KeyedMessage <string, string>(topic, topic, m)).ToArray();
messages[topic] = messageList;
Producer.Send(producerData);
builder.AddFetch(topic, partition, 0, 10000);
}
// wait a bit for produced message to be available
var request = builder.Build();
var response = Consumer.Fetch(request);
foreach (var topicAndPartition in topics)
{
var fetched = response.MessageSet(topicAndPartition.Item1, topicAndPartition.Item2);
Assert.Equal(messages[topicAndPartition.Item1], fetched.Select(m => Util.ReadString(m.Message.Payload)).ToList());
}
}
{
// send some invalid offsets
var builder = new FetchRequestBuilder();
foreach (var topicAndPartition in topics)
{
builder.AddFetch(topicAndPartition.Item1, topicAndPartition.Item2, -1, 10000);
}
try
{
var request = builder.Build();
var response = Consumer.Fetch(request);
foreach (var pdata in response.Data.Values)
{
ErrorMapping.MaybeThrowException(pdata.Error);
}
Assert.True(false, "Expected exception when fetching message with invalid offset");
}
catch (OffsetOutOfRangeException)
{
// ok
}
}
{
// send some invalid partitions
var builder = new FetchRequestBuilder();
foreach (var topicAndPartition in topics)
{
builder.AddFetch(topicAndPartition.Item1, -1, 0, 10000);
}
try
{
var request = builder.Build();
var response = Consumer.Fetch(request);
foreach (var pdata in response.Data.Values)
{
ErrorMapping.MaybeThrowException(pdata.Error);
}
Assert.True(false, "Expected exception when fetching message with invalid partition");
}
catch (UnknownTopicOrPartitionException)
{
// ok
}
}
}
/// <summary>
/// Method to be used for starting a new thread
/// </summary>
internal void Run()
{
foreach (PartitionTopicInfo partitionTopicInfo in _partitionTopicInfos)
{
Logger.InfoFormat("{0} start fetching topic: {1} part: {2} offset: {3} from {4}:{5}",
_name,
partitionTopicInfo.Topic,
partitionTopicInfo.PartitionId,
partitionTopicInfo.NextRequestOffset,
_broker.Host,
_broker.Port);
}
int reqId = 0;
while (!_shouldStop && _partitionTopicInfos.Any())
{
try
{
IEnumerable<PartitionTopicInfo> fetchablePartitionTopicInfos = _partitionTopicInfos.Where(pti => pti.GetMessagesCount() < _fetchBufferLength);
long read = 0;
if (fetchablePartitionTopicInfos.Any())
{
FetchRequestBuilder builder =
new FetchRequestBuilder().
CorrelationId(reqId).
ClientId(_config.ConsumerId ?? _name).
MaxWait(_config.MaxFetchWaitMs).
MinBytes(_config.FetchMinBytes);
fetchablePartitionTopicInfos.ForEach(pti => builder.AddFetch(pti.Topic, pti.PartitionId, pti.NextRequestOffset, _config.FetchSize));
FetchRequest fetchRequest = builder.Build();
Logger.Debug("Sending fetch request: " + fetchRequest);
FetchResponse response = _simpleConsumer.Fetch(fetchRequest);
Logger.Debug("Fetch request completed");
var partitonsWithErrors = new List<PartitionTopicInfo>();
foreach (PartitionTopicInfo partitionTopicInfo in fetchablePartitionTopicInfos)
{
BufferedMessageSet messages = response.MessageSet(partitionTopicInfo.Topic, partitionTopicInfo.PartitionId);
switch (messages.ErrorCode)
{
case (short)ErrorMapping.NoError:
int bytesRead = partitionTopicInfo.Add(messages);
// TODO: The highwater offset on the message set is the end of the log partition. If the message retrieved is -1 of that offset, we are at the end.
if (messages.Messages.Any())
{
partitionTopicInfo.NextRequestOffset = messages.Messages.Last().Offset + 1;
read += bytesRead;
}
else
{
Logger.DebugFormat("No message returned by FetchRequest: {0}", fetchRequest.ToString());
}
break;
case (short)ErrorMapping.OffsetOutOfRangeCode:
try
{
Logger.InfoFormat("offset for {0} out of range", partitionTopicInfo);
long resetOffset = ResetConsumerOffsets(partitionTopicInfo.Topic,
partitionTopicInfo.PartitionId);
if (resetOffset >= 0)
{
partitionTopicInfo.ResetOffset(resetOffset);
Logger.InfoFormat("{0} marked as done.", partitionTopicInfo);
}
}
catch (Exception ex)
{
Logger.ErrorFormat("Error getting offsets for partition {0} : {1}", partitionTopicInfo.PartitionId, ex.FormatException());
partitonsWithErrors.Add(partitionTopicInfo);
}
break;
default:
Logger.ErrorFormat("Error returned from broker {2} for partition {0} : KafkaErrorCode: {1}", partitionTopicInfo.PartitionId, messages.ErrorCode, partitionTopicInfo.BrokerId);
partitonsWithErrors.Add(partitionTopicInfo);
break;
}
}
reqId = reqId == int.MaxValue ? 0 : reqId + 1;
if (partitonsWithErrors.Any())
{
RemovePartitionsFromProessing(partitonsWithErrors);
}
}
if (read > 0)
{
Logger.Debug("Fetched bytes: " + read);
}
if (read == 0)
{
Logger.DebugFormat("backing off {0} ms", _config.BackOffIncrement);
Thread.Sleep(_config.BackOffIncrement);
}
}
catch (Exception ex)
{
if (_shouldStop)
{
Logger.InfoFormat("FetcherRunnable {0} interrupted", this);
}
else
{
Logger.ErrorFormat("error in FetcherRunnable {0}", ex.FormatException());
}
}
}
Logger.InfoFormat("stopping fetcher {0} to host {1}", _name, _broker.Host);
}
