private void RefreshMetadataInternal(short versionId, string clientId, int correlationId, string topic, Dictionary <string, TopicMetadata> tempTopicMetadatas, Dictionary <string, DateTime> tempTopicMetadatasLastUpdateTime, Dictionary <int, Tuple <Broker, BrokerConfiguration> > partitionLeaders) { Logger.InfoFormat("RefreshMetadataInternal enter: {0} {1} {2} Topic:{3} ", versionId, clientId, correlationId, topic); lock (syncProducerPoolForMetadataLock) { var brokerPartitionInfo = new BrokerPartitionInfo(syncProducerPoolForMetaData, tempTopicMetadatas, tempTopicMetadatasLastUpdateTime, ProducerConfiguration.DefaultTopicMetaDataRefreshIntervalMS, syncProducerPoolForMetaData.zkClient); brokerPartitionInfo.UpdateInfo(versionId, correlationId, clientId, topic); } if (!tempTopicMetadatas.ContainsKey(topic)) { throw new NoBrokerForTopicException( string.Format( "There is no metadata for topic {0}. Please check if all brokers of that topic live.", topic)); } var metadata = tempTopicMetadatas[topic]; if (metadata.Error != ErrorMapping.NoError) { throw new KafkaException( string.Format("The metadata status for topic {0} is abnormal, detail: ", topic), metadata.Error); ; } foreach (var p in metadata.PartitionsMetadata) { if (p.Leader != null && !partitionLeaders.ContainsKey(p.PartitionId)) { partitionLeaders.Add(p.PartitionId, new Tuple <Broker, BrokerConfiguration>( p.Leader, new BrokerConfiguration { BrokerId = p.Leader.Id, Host = p.Leader.Host, Port = p.Leader.Port })); Logger.DebugFormat("RefreshMetadataInternal Topic {0} partition {1} has leader {2}", topic, p.PartitionId, p.Leader.Id); } if (p.Leader == null) { Logger.ErrorFormat("RefreshMetadataInternal Topic {0} partition {1} does not have a leader yet.", topic, p.PartitionId); } } Logger.InfoFormat("RefreshMetadataInternal exit: {0} {1} {2} Topic:{3} ", versionId, clientId, correlationId, topic); }
public void Handle(IEnumerable <KeyedMessage <TKey, TValue> > events) { var serializedData = this.Serialize(events); foreach (var keyed in serializedData) { var dataSize = keyed.Message.PayloadSize; this.producerTopicStats.GetProducerTopicStats(keyed.Topic).ByteRate.Mark(dataSize); this.producerTopicStats.GetProducerAllTopicsStats().ByteRate.Mark(dataSize); } var outstandingProduceRequests = serializedData; var remainingRetries = this.Config.MessageSendMaxRetries + 1; var correlationIdStart = this.correlationId.Get(); Logger.DebugFormat("Handling {0} events", events.Count()); while (remainingRetries > 0 && outstandingProduceRequests.Any()) { foreach (var el in outstandingProduceRequests) { this.topicMetadataToRefresh.Add(el.Topic); } if (this.topicMetadataRefreshInterval >= TimeSpan.MinValue && (DateTime.Now - this.lastTopicMetadataRefeshTime) > this.topicMetadataRefreshInterval) { Util.SwallowError( Logger, () => this.brokerPartitionInfo.UpdateInfo( new HashSet <string>(this.topicMetadataToRefresh), this.correlationId.GetAndIncrement())); this.sendPartitionPerTopicCache.Clear(); this.topicMetadataToRefresh.Clear(); this.lastTopicMetadataRefeshTime = DateTime.Now; } outstandingProduceRequests = this.DispatchSerializedData(outstandingProduceRequests); if (outstandingProduceRequests.Any()) { Logger.InfoFormat("Back off for {0} ms before retrying send. Remaining retries = {1}", this.Config.RetryBackoffMs, remainingRetries - 1); // back off and update the topic metadata cache before attempting another send operation Thread.Sleep(this.Config.RetryBackoffMs); Util.SwallowError( Logger, () => { brokerPartitionInfo.UpdateInfo( new HashSet <string>(outstandingProduceRequests.Select(r => r.Topic)), correlationId.GetAndIncrement()); sendPartitionPerTopicCache.Clear(); remainingRetries -= 1; producerStats.ResendRate.Mark(); }); } this.sendPartitionPerTopicCache.Clear(); remainingRetries -= 1; } if (outstandingProduceRequests.Any()) { this.producerStats.FailedSendRate.Mark(); var correlationIdEnd = this.correlationId.Get(); Logger.ErrorFormat("Failed to send requests for topics {0} with correlation ids in [{1}, {2}]", string.Join(",", outstandingProduceRequests.Select(r => r.Topic)), correlationIdStart, correlationIdEnd); throw new FailedToSendMessageException( "Failed to send messages after " + this.Config.MessageSendMaxRetries + " tries"); } }