public PartitionData(int partition, ErrorMapping error, BufferedMessageSet messages) { Partition = partition; MessageSet = messages; Error = error; HighWaterMark = messages.HighwaterOffset; }
public PartitionData(int partition, ErrorMapping error, BufferedMessageSet messages) { this.Partition = partition; this.MessageSet = messages; this.Error = error; this.HighWaterMark = messages.HighwaterOffset; }
public void TestProduceAndMultiFetch() { // send some messages, with non-ordered topics var topicOffsets = new List <Tuple <string, int> > { Tuple.Create("test4", 0), Tuple.Create("test1", 0), Tuple.Create("test2", 0), Tuple.Create("test3", 0) }; { var messages = new Dictionary <string, List <string> >(); var builder = new FetchRequestBuilder(); foreach (var topicAndOffset in topicOffsets) { var topic = topicAndOffset.Item1; var offset = topicAndOffset.Item2; var producedData = new List <string> { "a_" + topic, "b_" + topic }; messages[topic] = producedData; Producer.Send(producedData.Select(m => new KeyedMessage <string, string>(topic, topic, m)).ToArray()); TestUtils.WaitUntilMetadataIsPropagated(this.Servers, topic, 0, 1000); builder.AddFetch(topic, offset, 0, 10000); } // wait a bit for produced message to be available var request = builder.Build(); var response = Consumer.Fetch(request); foreach (var topicAndOffset in topicOffsets) { var fetched = response.MessageSet(topicAndOffset.Item1, topicAndOffset.Item2); Assert.Equal(messages[topicAndOffset.Item1], fetched.Select(m => Util.ReadString(m.Message.Payload)).ToList()); } } { // send some invalid offsets var builder = new FetchRequestBuilder(); foreach (var topicAndOffset in topicOffsets) { builder.AddFetch(topicAndOffset.Item1, topicAndOffset.Item2, -1, 10000); } var request = builder.Build(); var responses = Consumer.Fetch(request); foreach (var pd in responses.Data.Values) { try { ErrorMapping.MaybeThrowException(pd.Error); Assert.True(false, "Expected an OffsetOutOfRangeException exception to be thrown"); } catch (OffsetOutOfRangeException) { // this is good } } } }
public static string GetMessage(ErrorMapping error) { switch (error) { case ErrorMapping.NoError: return("NoError"); case ErrorMapping.OffsetOutOfRangeCode: return("Offset out of range."); case ErrorMapping.InvalidMessageCode: return("Invalid message."); case ErrorMapping.MessagesizeTooLargeCode: return("Message size too large."); case ErrorMapping.UnknownTopicOrPartitionCode: return("Unknown topic or partition."); case ErrorMapping.InvalidFetchSizeCode: return("Invalid fetch size passed."); case ErrorMapping.LeaderNotAvailableCode: return("Leader not found for given topic/partition."); case ErrorMapping.NotLeaderForPartitionCode: return("This error is thrown if the client attempts to send messages to a replica that is not the leader for some partition. It indicates that the clients metadata is out of date."); case ErrorMapping.RequestTimedOutCode: return("This error is thrown if the request exceeds the user-specified time limit in the request."); case ErrorMapping.BrokerNotAvailableCode: return("This is not a client facing error and is used mostly by tools when a broker is not alive."); case ErrorMapping.ReplicaNotAvailableCode: return("If replica is expected on a broker, but is not."); case ErrorMapping.StaleControllerEpochCode: return("Internal error code for broker-to-broker communication."); case ErrorMapping.OffsetMetadataTooLargeCode: return("The server has a configurable maximum message size to avoid unbounded memory allocation. This error is thrown if the client attempt to produce a message larger than this maximum."); case ErrorMapping.StaleLeaderEpochCode: return("StaleLeaderEpochCode"); case ErrorMapping.OffsetsLoadInProgressCode: return("The broker returns this error code for an offset fetch request if it is still loading offsets (after a leader change for that offsets topic partition)."); case ErrorMapping.ConsumerCoordinatorNotAvailableCode: return("The broker returns this error code for consumer metadata requests or offset commit requests if the offsets topic has not yet been created."); case ErrorMapping.NotCoordinatorForConsumerCode: return("The broker returns this error code if it receives an offset fetch or commit request for a consumer group that it is not a coordinator for."); } return("Unknown Error."); }
public override string ToString() { var topicMetadataInfo = new StringBuilder(); topicMetadataInfo.AppendFormat("[TopicMetadata for topic {0} -> ", this.Topic); switch (this.ErrorCode) { case ErrorMapping.NoError: this.PartitionsMetadata.ForEach(partitionMetadata => { switch (partitionMetadata.ErrorCode) { case ErrorMapping.NoError: topicMetadataInfo.AppendFormat( " Metadata for partition [{0},{1}] is {2}", this.Topic, partitionMetadata.PartitionId, partitionMetadata.ToString()); break; case ErrorMapping.ReplicaNotAvailableCode: // this error message means some replica other than the leader is not available. The consumer // doesn't care about non leader replicas, so ignore this topicMetadataInfo.AppendFormat( " Metadata for partition [{0},{1}] is {2}", this.Topic, partitionMetadata.PartitionId, partitionMetadata.ToString()); break; default: topicMetadataInfo.AppendFormat( " Metadata for partition [{0},{1}] is not available due to {2}", this.Topic, partitionMetadata.PartitionId, ErrorMapping.ExceptionFor(partitionMetadata.ErrorCode).GetType().Name); break; } }); break; default: topicMetadataInfo.AppendFormat( "No partiton metadata for topic {0} due to {1}", this.Topic, ErrorMapping.ExceptionFor(this.ErrorCode).GetType().Name); break; } topicMetadataInfo.Append("]"); return(topicMetadataInfo.ToString()); }
/// <summary> /// Return a sequence of (brokerId, numPartitions). /// </summary> /// <param name="topic">the topic for which this information is to be returned</param> /// <param name="correlationId"></param> /// <returns></returns> public List <PartitionAndLeader> GetBrokerPartitionInfo(string topic, int correlationId) { Logger.DebugFormat("Getting broker partition info for topic {0}", topic); // check if the cache has metadata for this topic if (!this.topicPartitionInfo.ContainsKey(topic)) { // refresh the topic metadata cache this.UpdateInfo(new HashSet <string> { topic }, correlationId); if (!this.topicPartitionInfo.ContainsKey(topic)) { throw new KafkaException(string.Format("Failed to fetch topic metadata for topic: {0}", topic)); } } var metadata = this.topicPartitionInfo.Get(topic); var partitionMetadata = metadata.PartitionsMetadata; if (!partitionMetadata.Any()) { if (metadata.ErrorCode != ErrorMapping.NoError) { throw new KafkaException("Unable to get broker partition info", ErrorMapping.ExceptionFor(metadata.ErrorCode)); } else { throw new KafkaException(string.Format("Topic metadata {0} has empty partition metadata and no error code", metadata)); } } return(partitionMetadata.Select(m => { if (m.Leader != null) { Logger.DebugFormat("Partition [{0}, {1}] has leader {2}", topic, m.PartitionId, m.Leader.Id); return new PartitionAndLeader(topic, m.PartitionId, m.Leader.Id); } else { Logger.DebugFormat( "Partition [{0}, {1}] does not have a leader yet", topic, m.PartitionId); return new PartitionAndLeader(topic, m.PartitionId, null); } }).OrderBy(x => x.PartitionId).ToList()); }
public void TestProduceAndFetch() { // send some messages var topic = "test"; var sendMessages = new List <string> { "hello", "there" }; var producerData = sendMessages.Select(m => new KeyedMessage <string, string>(topic, topic, m)).ToArray(); Producer.Send(producerData); TestUtils.WaitUntilMetadataIsPropagated(this.Servers, topic, 0, 1000); ByteBufferMessageSet fetchedMessage = null; while (fetchedMessage == null || fetchedMessage.ValidBytes == 0) { var fetched = Consumer.Fetch(new FetchRequestBuilder().AddFetch(topic, 0, 0, 10000).Build()); fetchedMessage = fetched.MessageSet(topic, 0); } Assert.Equal(sendMessages, fetchedMessage.Select(m => Util.ReadString(m.Message.Payload)).ToList()); // send an invalid offset try { var fetchedWithError = Consumer.Fetch(new FetchRequestBuilder().AddFetch(topic, 0, -1, 10000).Build()); foreach (var pdata in fetchedWithError.Data.Values) { ErrorMapping.MaybeThrowException(pdata.Error); } Assert.True(false, "Expected an OffsetOutOfRangeException exception to be thrown"); } catch (OffsetOutOfRangeException) { } }
public PartitionOffsetsResponse(int partitionId, ErrorMapping error, List <long> offsets) { PartitionId = partitionId; Error = error; Offsets = offsets; }
public void TestProduceAndMultiFetch() { this.CreateSimpleTopicsAndAwaitLeader( this.ZkClient, new List <string> { "test1", "test2", "test3", "test4" }, Configs.First().BrokerId); // send some messages, with non-ordered topics var topics = new List <Tuple <string, int> > { Tuple.Create("test4", 0), Tuple.Create("test1", 0), Tuple.Create("test2", 0), Tuple.Create("test3", 0) }; { var messages = new Dictionary <string, List <string> >(); var builder = new FetchRequestBuilder(); foreach (var topicAndOffset in topics) { var topic = topicAndOffset.Item1; var partition = topicAndOffset.Item2; var messageList = new List <string> { "a_" + topic, "b_" + topic }; var producerData = messageList.Select(m => new KeyedMessage <string, string>(topic, topic, m)).ToArray(); messages[topic] = messageList; Producer.Send(producerData); builder.AddFetch(topic, partition, 0, 10000); } // wait a bit for produced message to be available var request = builder.Build(); var response = Consumer.Fetch(request); foreach (var topicAndPartition in topics) { var fetched = response.MessageSet(topicAndPartition.Item1, topicAndPartition.Item2); Assert.Equal(messages[topicAndPartition.Item1], fetched.Select(m => Util.ReadString(m.Message.Payload)).ToList()); } } { // send some invalid offsets var builder = new FetchRequestBuilder(); foreach (var topicAndPartition in topics) { builder.AddFetch(topicAndPartition.Item1, topicAndPartition.Item2, -1, 10000); } try { var request = builder.Build(); var response = Consumer.Fetch(request); foreach (var pdata in response.Data.Values) { ErrorMapping.MaybeThrowException(pdata.Error); } Assert.True(false, "Expected exception when fetching message with invalid offset"); } catch (OffsetOutOfRangeException) { // ok } } { // send some invalid partitions var builder = new FetchRequestBuilder(); foreach (var topicAndPartition in topics) { builder.AddFetch(topicAndPartition.Item1, -1, 0, 10000); } try { var request = builder.Build(); var response = Consumer.Fetch(request); foreach (var pdata in response.Data.Values) { ErrorMapping.MaybeThrowException(pdata.Error); } Assert.True(false, "Expected exception when fetching message with invalid partition"); } catch (UnknownTopicOrPartitionException) { // ok } } }
/// <summary> /// Constructs and sends the produce request based on a map from (topic, partition) -> messages /// </summary> /// <param name="brokerId">brokerId the broker that will receive the request</param> /// <param name="messagesPerTopic"></param> /// <returns> the set (topic, partitions) messages which incurred an error sending or processing</returns> private List <TopicAndPartition> Send(int brokerId, IDictionary <TopicAndPartition, ByteBufferMessageSet> messagesPerTopic) { if (brokerId < 0) { Logger.WarnFormat("Failed to send Data since partitions {0} don't have a leader", string.Join(",", messagesPerTopic.Select(m => m.Key.Partiton))); return(new List <TopicAndPartition>(messagesPerTopic.Keys)); } if (messagesPerTopic.Count > 0) { var currentCorrelationId = this.correlationId.GetAndIncrement(); var producerRequest = new ProducerRequest(currentCorrelationId, this.Config.ClientId, this.Config.RequestRequiredAcks, this.Config.RequestTimeoutMs, messagesPerTopic); var failedTopicPartitions = new List <TopicAndPartition>(); try { var syncProducer = this.producerPool.GetProducer(brokerId); Logger.DebugFormat( "Producer sending messages with correlation id {0} for topics {1} to broker {2} on {3}:{4}", currentCorrelationId, string.Join(",", messagesPerTopic.Keys), brokerId, syncProducer.Config.Host, syncProducer.Config.Port); var response = syncProducer.Send(producerRequest); if (response != null) { if (response.Status.Count() != producerRequest.Data.Count()) { throw new KafkaException( string.Format( "Incomplete response ({0}) for producer request ({1})", response, producerRequest)); } if (Logger.IsDebugEnabled) { var successfullySentData = response.Status.Where(s => s.Value.Error == ErrorMapping.NoError).ToList(); foreach (var m in successfullySentData) { var iter = messagesPerTopic[m.Key].Iterator(); while (iter.HasNext()) { var message = iter.Next(); Logger.DebugFormat( "Successfully sent messsage: {0}", message.Message.IsNull() ? null : Util.ReadString(message.Message.Payload)); } } } var failedPartitionsAndStatus = response.Status.Where(s => s.Value.Error != ErrorMapping.NoError).ToList(); failedTopicPartitions = failedPartitionsAndStatus.Select(partitionStatus => partitionStatus.Key).ToList(); if (failedTopicPartitions.Any()) { var errorString = string.Join( ",", failedPartitionsAndStatus.OrderBy(x => x.Key.Topic) .ThenBy(x => x.Key.Partiton) .Select( kvp => { var topicAndPartiton = kvp.Key; var status = kvp.Value; return(topicAndPartiton.ToString() + ": " + ErrorMapping.ExceptionFor(status.Error) .GetType() .Name); })); Logger.WarnFormat("Produce request with correlation id {0} failed due to {1}", currentCorrelationId, errorString); } return(failedTopicPartitions); } else { return(new List <TopicAndPartition>()); } } catch (Exception e) { Logger.Warn( string.Format( "Failed to send producer request with correlation id {0} to broker {1} with Data for partitions {2}", currentCorrelationId, brokerId, string.Join(",", messagesPerTopic.Select(m => m.Key))), e); return(new List <TopicAndPartition>(messagesPerTopic.Keys)); } } else { return(new List <TopicAndPartition>()); } }
public override string ToString() { return(string.Format("Error: {0}, Offsets: {1}", ErrorMapping.ExceptionFor(this.Error), string.Join(",", this.Offsets))); }
public KafkaException(string message, ErrorMapping errorCode) : base(message + GetMessage((short)errorCode)) { ErrorCode = (short)errorCode; }
public KafkaClientException(string message, ErrorMapping errorCode) : this(message) { ErrorCode = errorCode; }
public TopicMetadata(string topic, IEnumerable <PartitionMetadata> partitionsMetadata, ErrorMapping error) { Topic = topic; PartitionsMetadata = partitionsMetadata; Error = error; }
/// <summary> /// It updates the cache by issuing a get topic metadata request to a random broker. /// </summary> /// <param name="topics"></param> /// <param name="correlationId"></param> public void UpdateInfo(ISet <string> topics, int correlationId) { List <TopicMetadata> topicsMetadata; var topicMetadataResponse = ClientUtils.FetchTopicMetadata(topics, this.brokers, this.producerConfig, correlationId); topicsMetadata = topicMetadataResponse.TopicsMetadata; foreach (var tmd in topicsMetadata) { Logger.DebugFormat("Metadata for topic {0} is {1}", tmd.Topic, tmd); if (tmd.ErrorCode == ErrorMapping.NoError) { this.topicPartitionInfo[tmd.Topic] = tmd; } else { Logger.WarnFormat("Error while fetch metadata [{0}] for topic [{1}]: {2}", tmd, tmd.Topic, ErrorMapping.ExceptionFor(tmd.ErrorCode).GetType().Name); foreach (var pmd in tmd.PartitionsMetadata) { if (pmd.ErrorCode != ErrorMapping.NoError && pmd.ErrorCode == ErrorMapping.LeaderNotAvailableCode) { Logger.WarnFormat("Error while fetching metadata {0} for topic partiton [{1},{2}]:[{3}]", pmd, tmd.Topic, pmd.PartitionId, ErrorMapping.ExceptionFor(pmd.ErrorCode).GetType()); //// any other error code (e.g. ReplicaNotAvailable) can be ignored since the producer does not need to access the replica and isr metadata } } } } this.producerPool.UpdateProducer(topicsMetadata); }
public void ProcessFetchRequest(FetchRequest fetchRequest) { var partitionsWithError = new HashSet <TopicAndPartition>(); FetchResponse response = null; try { Logger.DebugFormat("issuing to broker {0} of fetch request {1}", this.sourceBroker.Id, fetchRequest); response = this.simpleConsumer.Fetch(fetchRequest); } catch (Exception e) { if (isRunning.Get()) { Logger.Error("Error in fetch " + fetchRequest, e); this.partitionMapLock.Lock(); try { foreach (var key in this.partitionMap.Keys) { partitionsWithError.Add(key); } } finally { this.partitionMapLock.Unlock(); } } } this.FetcherStats.RequestRate.Mark(); if (response != null) { // process fetched Data this.partitionMapLock.Lock(); try { foreach (var topicAndData in response.Data) { var topicAndPartition = topicAndData.Key; var partitionData = topicAndData.Value; var topic = topicAndPartition.Topic; var partitionId = topicAndPartition.Partiton; long currentOffset; if (this.partitionMap.TryGetValue(topicAndPartition, out currentOffset) && fetchRequest.RequestInfo[topicAndPartition].Offset == currentOffset) { // we append to the log if the current offset is defined and it is the same as the offset requested during fetch switch (partitionData.Error) { case ErrorMapping.NoError: try { var messages = (ByteBufferMessageSet)partitionData.Messages; var validBytes = messages.ValidBytes; var messageAndOffset = messages.ShallowIterator().ToEnumerable().LastOrDefault(); var newOffset = messageAndOffset != null ? messageAndOffset.NextOffset : currentOffset; this.partitionMap[topicAndPartition] = newOffset; this.FetcherLagStats.GetFetcherLagStats(topic, partitionId).Lag = partitionData.Hw - newOffset; this.FetcherStats.ByteRate.Mark(validBytes); // Once we hand off the partition Data to the subclass, we can't mess with it any more in this thread this.ProcessPartitionData(topicAndPartition, currentOffset, partitionData); } catch (InvalidMessageException ime) { // we log the error and continue. This ensures two things // 1. If there is a corrupt message in a topic partition, it does not bring the fetcher thread down and cause other topic partition to also lag // 2. If the message is corrupt due to a transient state in the log (truncation, partial writes can cause this), we simply continue and // should get fixed in the subsequent fetches Logger.ErrorFormat( "Found invalid messages during fetch for partiton [{0},{1}] offset {2} error {3}", topic, partitionId, currentOffset, ime.Message); } catch (Exception e) { throw new KafkaException( string.Format( "error processing Data for partition [{0},{1}] offset {2}", topic, partitionId, currentOffset), e); } break; case ErrorMapping.OffsetOutOfRangeCode: try { var newOffset = this.HandleOffsetOutOfRange(topicAndPartition); this.partitionMap[topicAndPartition] = newOffset; Logger.ErrorFormat( "Current offset {0} for partiton [{1},{2}] out of range; reste offset to {3}", currentOffset, topic, partitionId, newOffset); } catch (Exception e) { Logger.Error( string.Format( "Error getting offset for partiton [{0},{1}] to broker {2}", topic, partitionId, sourceBroker.Id), e); partitionsWithError.Add(topicAndPartition); } break; default: if (isRunning.Get()) { Logger.ErrorFormat( "Error for partition [{0},{1}] to broker {2}:{3}", topic, partitionId, this.sourceBroker.Id, ErrorMapping.ExceptionFor(partitionData.Error).GetType().Name); partitionsWithError.Add(topicAndPartition); } break; } } } } finally { this.partitionMapLock.Unlock(); } } if (partitionsWithError.Count > 0) { Logger.DebugFormat("handling partitions with error for {0}", string.Join(",", partitionsWithError)); this.HandlePartitionsWithErrors(partitionsWithError); } }
public KafkaConsumeException(string message, ErrorMapping errorCode) : base(message, errorCode) { }
public KafkaException(ErrorMapping errorCode) : base(GetMessage((short)errorCode)) { ErrorCode = (short)errorCode; }