public PartitionData(int partition, ErrorMapping error, BufferedMessageSet messages)
{
Partition = partition;
MessageSet = messages;
Error = error;
HighWaterMark = messages.HighwaterOffset;
}
public PartitionData(int partition, ErrorMapping error, BufferedMessageSet messages)
{
this.Partition = partition;
this.MessageSet = messages;
this.Error = error;
this.HighWaterMark = messages.HighwaterOffset;
}
public void TestProduceAndMultiFetch()
{
// send some messages, with non-ordered topics
var topicOffsets = new List <Tuple <string, int> > {
Tuple.Create("test4", 0), Tuple.Create("test1", 0), Tuple.Create("test2", 0), Tuple.Create("test3", 0)
};
{
var messages = new Dictionary <string, List <string> >();
var builder = new FetchRequestBuilder();
foreach (var topicAndOffset in topicOffsets)
{
var topic = topicAndOffset.Item1;
var offset = topicAndOffset.Item2;
var producedData = new List <string> {
"a_" + topic, "b_" + topic
};
messages[topic] = producedData;
Producer.Send(producedData.Select(m => new KeyedMessage <string, string>(topic, topic, m)).ToArray());
TestUtils.WaitUntilMetadataIsPropagated(this.Servers, topic, 0, 1000);
builder.AddFetch(topic, offset, 0, 10000);
}
// wait a bit for produced message to be available
var request = builder.Build();
var response = Consumer.Fetch(request);
foreach (var topicAndOffset in topicOffsets)
{
var fetched = response.MessageSet(topicAndOffset.Item1, topicAndOffset.Item2);
Assert.Equal(messages[topicAndOffset.Item1], fetched.Select(m => Util.ReadString(m.Message.Payload)).ToList());
}
}
{
// send some invalid offsets
var builder = new FetchRequestBuilder();
foreach (var topicAndOffset in topicOffsets)
{
builder.AddFetch(topicAndOffset.Item1, topicAndOffset.Item2, -1, 10000);
}
var request = builder.Build();
var responses = Consumer.Fetch(request);
foreach (var pd in responses.Data.Values)
{
try
{
ErrorMapping.MaybeThrowException(pd.Error);
Assert.True(false, "Expected an OffsetOutOfRangeException exception to be thrown");
}
catch (OffsetOutOfRangeException)
{
// this is good
}
}
}
}
public static string GetMessage(ErrorMapping error)
{
switch (error)
{
case ErrorMapping.NoError:
return("NoError");
case ErrorMapping.OffsetOutOfRangeCode:
return("Offset out of range.");
case ErrorMapping.InvalidMessageCode:
return("Invalid message.");
case ErrorMapping.MessagesizeTooLargeCode:
return("Message size too large.");
case ErrorMapping.UnknownTopicOrPartitionCode:
return("Unknown topic or partition.");
case ErrorMapping.InvalidFetchSizeCode:
return("Invalid fetch size passed.");
case ErrorMapping.LeaderNotAvailableCode:
return("Leader not found for given topic/partition.");
case ErrorMapping.NotLeaderForPartitionCode:
return("This error is thrown if the client attempts to send messages to a replica that is not the leader for some partition. It indicates that the clients metadata is out of date.");
case ErrorMapping.RequestTimedOutCode:
return("This error is thrown if the request exceeds the user-specified time limit in the request.");
case ErrorMapping.BrokerNotAvailableCode:
return("This is not a client facing error and is used mostly by tools when a broker is not alive.");
case ErrorMapping.ReplicaNotAvailableCode:
return("If replica is expected on a broker, but is not.");
case ErrorMapping.StaleControllerEpochCode:
return("Internal error code for broker-to-broker communication.");
case ErrorMapping.OffsetMetadataTooLargeCode:
return("The server has a configurable maximum message size to avoid unbounded memory allocation. This error is thrown if the client attempt to produce a message larger than this maximum.");
case ErrorMapping.StaleLeaderEpochCode:
return("StaleLeaderEpochCode");
case ErrorMapping.OffsetsLoadInProgressCode:
return("The broker returns this error code for an offset fetch request if it is still loading offsets (after a leader change for that offsets topic partition).");
case ErrorMapping.ConsumerCoordinatorNotAvailableCode:
return("The broker returns this error code for consumer metadata requests or offset commit requests if the offsets topic has not yet been created.");
case ErrorMapping.NotCoordinatorForConsumerCode:
return("The broker returns this error code if it receives an offset fetch or commit request for a consumer group that it is not a coordinator for.");
}
return("Unknown Error.");
}
public override string ToString()
{
var topicMetadataInfo = new StringBuilder();
topicMetadataInfo.AppendFormat("[TopicMetadata for topic {0} -> ", this.Topic);
switch (this.ErrorCode)
{
case ErrorMapping.NoError:
this.PartitionsMetadata.ForEach(partitionMetadata =>
{
switch (partitionMetadata.ErrorCode)
{
case ErrorMapping.NoError:
topicMetadataInfo.AppendFormat(
" Metadata for partition [{0},{1}] is {2}",
this.Topic,
partitionMetadata.PartitionId,
partitionMetadata.ToString());
break;
case ErrorMapping.ReplicaNotAvailableCode:
// this error message means some replica other than the leader is not available. The consumer
// doesn't care about non leader replicas, so ignore this
topicMetadataInfo.AppendFormat(
" Metadata for partition [{0},{1}] is {2}",
this.Topic,
partitionMetadata.PartitionId,
partitionMetadata.ToString());
break;
default:
topicMetadataInfo.AppendFormat(
" Metadata for partition [{0},{1}] is not available due to {2}",
this.Topic,
partitionMetadata.PartitionId,
ErrorMapping.ExceptionFor(partitionMetadata.ErrorCode).GetType().Name);
break;
}
});
break;
default:
topicMetadataInfo.AppendFormat(
"No partiton metadata for topic {0} due to {1}",
this.Topic,
ErrorMapping.ExceptionFor(this.ErrorCode).GetType().Name);
break;
}
topicMetadataInfo.Append("]");
return(topicMetadataInfo.ToString());
}
/// <summary>
/// Return a sequence of (brokerId, numPartitions).
/// </summary>
/// <param name="topic">the topic for which this information is to be returned</param>
/// <param name="correlationId"></param>
/// <returns></returns>
public List <PartitionAndLeader> GetBrokerPartitionInfo(string topic, int correlationId)
{
Logger.DebugFormat("Getting broker partition info for topic {0}", topic);
// check if the cache has metadata for this topic
if (!this.topicPartitionInfo.ContainsKey(topic))
{
// refresh the topic metadata cache
this.UpdateInfo(new HashSet <string> {
topic
}, correlationId);
if (!this.topicPartitionInfo.ContainsKey(topic))
{
throw new KafkaException(string.Format("Failed to fetch topic metadata for topic: {0}", topic));
}
}
var metadata = this.topicPartitionInfo.Get(topic);
var partitionMetadata = metadata.PartitionsMetadata;
if (!partitionMetadata.Any())
{
if (metadata.ErrorCode != ErrorMapping.NoError)
{
throw new KafkaException("Unable to get broker partition info", ErrorMapping.ExceptionFor(metadata.ErrorCode));
}
else
{
throw new KafkaException(string.Format("Topic metadata {0} has empty partition metadata and no error code", metadata));
}
}
return(partitionMetadata.Select(m =>
{
if (m.Leader != null)
{
Logger.DebugFormat("Partition [{0}, {1}] has leader {2}", topic, m.PartitionId, m.Leader.Id);
return new PartitionAndLeader(topic, m.PartitionId, m.Leader.Id);
}
else
{
Logger.DebugFormat(
"Partition [{0}, {1}] does not have a leader yet",
topic,
m.PartitionId);
return new PartitionAndLeader(topic, m.PartitionId, null);
}
}).OrderBy(x => x.PartitionId).ToList());
}
public void TestProduceAndFetch()
{
// send some messages
var topic = "test";
var sendMessages = new List <string> {
"hello", "there"
};
var producerData = sendMessages.Select(m => new KeyedMessage <string, string>(topic, topic, m)).ToArray();
Producer.Send(producerData);
TestUtils.WaitUntilMetadataIsPropagated(this.Servers, topic, 0, 1000);
ByteBufferMessageSet fetchedMessage = null;
while (fetchedMessage == null || fetchedMessage.ValidBytes == 0)
{
var fetched = Consumer.Fetch(new FetchRequestBuilder().AddFetch(topic, 0, 0, 10000).Build());
fetchedMessage = fetched.MessageSet(topic, 0);
}
Assert.Equal(sendMessages, fetchedMessage.Select(m => Util.ReadString(m.Message.Payload)).ToList());
// send an invalid offset
try
{
var fetchedWithError = Consumer.Fetch(new FetchRequestBuilder().AddFetch(topic, 0, -1, 10000).Build());
foreach (var pdata in fetchedWithError.Data.Values)
{
ErrorMapping.MaybeThrowException(pdata.Error);
}
Assert.True(false, "Expected an OffsetOutOfRangeException exception to be thrown");
}
catch (OffsetOutOfRangeException)
{
}
}
public PartitionOffsetsResponse(int partitionId, ErrorMapping error, List <long> offsets)
{
PartitionId = partitionId;
Error = error;
Offsets = offsets;
}
public void TestProduceAndMultiFetch()
{
this.CreateSimpleTopicsAndAwaitLeader(
this.ZkClient, new List <string> {
"test1", "test2", "test3", "test4"
}, Configs.First().BrokerId);
// send some messages, with non-ordered topics
var topics = new List <Tuple <string, int> > {
Tuple.Create("test4", 0), Tuple.Create("test1", 0), Tuple.Create("test2", 0), Tuple.Create("test3", 0)
};
{
var messages = new Dictionary <string, List <string> >();
var builder = new FetchRequestBuilder();
foreach (var topicAndOffset in topics)
{
var topic = topicAndOffset.Item1;
var partition = topicAndOffset.Item2;
var messageList = new List <string> {
"a_" + topic, "b_" + topic
};
var producerData = messageList.Select(m => new KeyedMessage <string, string>(topic, topic, m)).ToArray();
messages[topic] = messageList;
Producer.Send(producerData);
builder.AddFetch(topic, partition, 0, 10000);
}
// wait a bit for produced message to be available
var request = builder.Build();
var response = Consumer.Fetch(request);
foreach (var topicAndPartition in topics)
{
var fetched = response.MessageSet(topicAndPartition.Item1, topicAndPartition.Item2);
Assert.Equal(messages[topicAndPartition.Item1], fetched.Select(m => Util.ReadString(m.Message.Payload)).ToList());
}
}
{
// send some invalid offsets
var builder = new FetchRequestBuilder();
foreach (var topicAndPartition in topics)
{
builder.AddFetch(topicAndPartition.Item1, topicAndPartition.Item2, -1, 10000);
}
try
{
var request = builder.Build();
var response = Consumer.Fetch(request);
foreach (var pdata in response.Data.Values)
{
ErrorMapping.MaybeThrowException(pdata.Error);
}
Assert.True(false, "Expected exception when fetching message with invalid offset");
}
catch (OffsetOutOfRangeException)
{
// ok
}
}
{
// send some invalid partitions
var builder = new FetchRequestBuilder();
foreach (var topicAndPartition in topics)
{
builder.AddFetch(topicAndPartition.Item1, -1, 0, 10000);
}
try
{
var request = builder.Build();
var response = Consumer.Fetch(request);
foreach (var pdata in response.Data.Values)
{
ErrorMapping.MaybeThrowException(pdata.Error);
}
Assert.True(false, "Expected exception when fetching message with invalid partition");
}
catch (UnknownTopicOrPartitionException)
{
// ok
}
}
}
/// <summary>
/// Constructs and sends the produce request based on a map from (topic, partition) -> messages
/// </summary>
/// <param name="brokerId">brokerId the broker that will receive the request</param>
/// <param name="messagesPerTopic"></param>
/// <returns> the set (topic, partitions) messages which incurred an error sending or processing</returns>
private List <TopicAndPartition> Send(int brokerId, IDictionary <TopicAndPartition, ByteBufferMessageSet> messagesPerTopic)
{
if (brokerId < 0)
{
Logger.WarnFormat("Failed to send Data since partitions {0} don't have a leader", string.Join(",", messagesPerTopic.Select(m => m.Key.Partiton)));
return(new List <TopicAndPartition>(messagesPerTopic.Keys));
}
if (messagesPerTopic.Count > 0)
{
var currentCorrelationId = this.correlationId.GetAndIncrement();
var producerRequest = new ProducerRequest(currentCorrelationId, this.Config.ClientId, this.Config.RequestRequiredAcks, this.Config.RequestTimeoutMs, messagesPerTopic);
var failedTopicPartitions = new List <TopicAndPartition>();
try
{
var syncProducer = this.producerPool.GetProducer(brokerId);
Logger.DebugFormat(
"Producer sending messages with correlation id {0} for topics {1} to broker {2} on {3}:{4}",
currentCorrelationId,
string.Join(",", messagesPerTopic.Keys),
brokerId,
syncProducer.Config.Host,
syncProducer.Config.Port);
var response = syncProducer.Send(producerRequest);
if (response != null)
{
if (response.Status.Count() != producerRequest.Data.Count())
{
throw new KafkaException(
string.Format(
"Incomplete response ({0}) for producer request ({1})", response, producerRequest));
}
if (Logger.IsDebugEnabled)
{
var successfullySentData = response.Status.Where(s => s.Value.Error == ErrorMapping.NoError).ToList();
foreach (var m in successfullySentData)
{
var iter = messagesPerTopic[m.Key].Iterator();
while (iter.HasNext())
{
var message = iter.Next();
Logger.DebugFormat(
"Successfully sent messsage: {0}",
message.Message.IsNull() ? null : Util.ReadString(message.Message.Payload));
}
}
}
var failedPartitionsAndStatus = response.Status.Where(s => s.Value.Error != ErrorMapping.NoError).ToList();
failedTopicPartitions =
failedPartitionsAndStatus.Select(partitionStatus => partitionStatus.Key).ToList();
if (failedTopicPartitions.Any())
{
var errorString = string.Join(
",",
failedPartitionsAndStatus.OrderBy(x => x.Key.Topic)
.ThenBy(x => x.Key.Partiton)
.Select(
kvp =>
{
var topicAndPartiton = kvp.Key;
var status = kvp.Value;
return(topicAndPartiton.ToString() + ": "
+ ErrorMapping.ExceptionFor(status.Error)
.GetType()
.Name);
}));
Logger.WarnFormat("Produce request with correlation id {0} failed due to {1}", currentCorrelationId, errorString);
}
return(failedTopicPartitions);
}
else
{
return(new List <TopicAndPartition>());
}
}
catch (Exception e)
{
Logger.Warn(
string.Format(
"Failed to send producer request with correlation id {0} to broker {1} with Data for partitions {2}",
currentCorrelationId,
brokerId,
string.Join(",", messagesPerTopic.Select(m => m.Key))),
e);
return(new List <TopicAndPartition>(messagesPerTopic.Keys));
}
}
else
{
return(new List <TopicAndPartition>());
}
}
public override string ToString()
{
return(string.Format("Error: {0}, Offsets: {1}", ErrorMapping.ExceptionFor(this.Error), string.Join(",", this.Offsets)));
}
public KafkaException(string message, ErrorMapping errorCode)
: base(message + GetMessage((short)errorCode))
{
ErrorCode = (short)errorCode;
}
public KafkaClientException(string message, ErrorMapping errorCode) : this(message)
{
ErrorCode = errorCode;
}
public TopicMetadata(string topic, IEnumerable <PartitionMetadata> partitionsMetadata, ErrorMapping error)
{
Topic = topic;
PartitionsMetadata = partitionsMetadata;
Error = error;
}
/// <summary>
/// It updates the cache by issuing a get topic metadata request to a random broker.
/// </summary>
/// <param name="topics"></param>
/// <param name="correlationId"></param>
public void UpdateInfo(ISet <string> topics, int correlationId)
{
List <TopicMetadata> topicsMetadata;
var topicMetadataResponse = ClientUtils.FetchTopicMetadata(topics, this.brokers, this.producerConfig, correlationId);
topicsMetadata = topicMetadataResponse.TopicsMetadata;
foreach (var tmd in topicsMetadata)
{
Logger.DebugFormat("Metadata for topic {0} is {1}", tmd.Topic, tmd);
if (tmd.ErrorCode == ErrorMapping.NoError)
{
this.topicPartitionInfo[tmd.Topic] = tmd;
}
else
{
Logger.WarnFormat("Error while fetch metadata [{0}] for topic [{1}]: {2}", tmd, tmd.Topic, ErrorMapping.ExceptionFor(tmd.ErrorCode).GetType().Name);
foreach (var pmd in tmd.PartitionsMetadata)
{
if (pmd.ErrorCode != ErrorMapping.NoError &&
pmd.ErrorCode == ErrorMapping.LeaderNotAvailableCode)
{
Logger.WarnFormat("Error while fetching metadata {0} for topic partiton [{1},{2}]:[{3}]", pmd, tmd.Topic, pmd.PartitionId, ErrorMapping.ExceptionFor(pmd.ErrorCode).GetType());
//// any other error code (e.g. ReplicaNotAvailable) can be ignored since the producer does not need to access the replica and isr metadata
}
}
}
}
this.producerPool.UpdateProducer(topicsMetadata);
}
public void ProcessFetchRequest(FetchRequest fetchRequest)
{
var partitionsWithError = new HashSet <TopicAndPartition>();
FetchResponse response = null;
try
{
Logger.DebugFormat("issuing to broker {0} of fetch request {1}", this.sourceBroker.Id, fetchRequest);
response = this.simpleConsumer.Fetch(fetchRequest);
}
catch (Exception e)
{
if (isRunning.Get())
{
Logger.Error("Error in fetch " + fetchRequest, e);
this.partitionMapLock.Lock();
try
{
foreach (var key in this.partitionMap.Keys)
{
partitionsWithError.Add(key);
}
}
finally
{
this.partitionMapLock.Unlock();
}
}
}
this.FetcherStats.RequestRate.Mark();
if (response != null)
{
// process fetched Data
this.partitionMapLock.Lock();
try
{
foreach (var topicAndData in response.Data)
{
var topicAndPartition = topicAndData.Key;
var partitionData = topicAndData.Value;
var topic = topicAndPartition.Topic;
var partitionId = topicAndPartition.Partiton;
long currentOffset;
if (this.partitionMap.TryGetValue(topicAndPartition, out currentOffset) &&
fetchRequest.RequestInfo[topicAndPartition].Offset == currentOffset)
{
// we append to the log if the current offset is defined and it is the same as the offset requested during fetch
switch (partitionData.Error)
{
case ErrorMapping.NoError:
try
{
var messages = (ByteBufferMessageSet)partitionData.Messages;
var validBytes = messages.ValidBytes;
var messageAndOffset =
messages.ShallowIterator().ToEnumerable().LastOrDefault();
var newOffset = messageAndOffset != null
? messageAndOffset.NextOffset
: currentOffset;
this.partitionMap[topicAndPartition] = newOffset;
this.FetcherLagStats.GetFetcherLagStats(topic, partitionId).Lag = partitionData.Hw
- newOffset;
this.FetcherStats.ByteRate.Mark(validBytes);
// Once we hand off the partition Data to the subclass, we can't mess with it any more in this thread
this.ProcessPartitionData(topicAndPartition, currentOffset, partitionData);
}
catch (InvalidMessageException ime)
{
// we log the error and continue. This ensures two things
// 1. If there is a corrupt message in a topic partition, it does not bring the fetcher thread down and cause other topic partition to also lag
// 2. If the message is corrupt due to a transient state in the log (truncation, partial writes can cause this), we simply continue and
// should get fixed in the subsequent fetches
Logger.ErrorFormat(
"Found invalid messages during fetch for partiton [{0},{1}] offset {2} error {3}",
topic,
partitionId,
currentOffset,
ime.Message);
}
catch (Exception e)
{
throw new KafkaException(
string.Format(
"error processing Data for partition [{0},{1}] offset {2}",
topic,
partitionId,
currentOffset),
e);
}
break;
case ErrorMapping.OffsetOutOfRangeCode:
try
{
var newOffset = this.HandleOffsetOutOfRange(topicAndPartition);
this.partitionMap[topicAndPartition] = newOffset;
Logger.ErrorFormat(
"Current offset {0} for partiton [{1},{2}] out of range; reste offset to {3}",
currentOffset,
topic,
partitionId,
newOffset);
}
catch (Exception e)
{
Logger.Error(
string.Format(
"Error getting offset for partiton [{0},{1}] to broker {2}",
topic,
partitionId,
sourceBroker.Id),
e);
partitionsWithError.Add(topicAndPartition);
}
break;
default:
if (isRunning.Get())
{
Logger.ErrorFormat(
"Error for partition [{0},{1}] to broker {2}:{3}",
topic,
partitionId,
this.sourceBroker.Id,
ErrorMapping.ExceptionFor(partitionData.Error).GetType().Name);
partitionsWithError.Add(topicAndPartition);
}
break;
}
}
}
}
finally
{
this.partitionMapLock.Unlock();
}
}
if (partitionsWithError.Count > 0)
{
Logger.DebugFormat("handling partitions with error for {0}", string.Join(",", partitionsWithError));
this.HandlePartitionsWithErrors(partitionsWithError);
}
}
public KafkaConsumeException(string message, ErrorMapping errorCode)
: base(message, errorCode)
{
}
public KafkaException(ErrorMapping errorCode)
: base(GetMessage((short)errorCode))
{
ErrorCode = (short)errorCode;
}
