private void RefreshMetadataInternal(short versionId,
string clientId,
int correlationId,
string topic,
Dictionary <string, TopicMetadata> tempTopicMetadatas,
Dictionary <string, DateTime> tempTopicMetadatasLastUpdateTime,
Dictionary <int, Tuple <Broker, BrokerConfiguration> > partitionLeaders)
{
Logger.InfoFormat("RefreshMetadataInternal enter: {0} {1} {2} Topic:{3} ", versionId, clientId,
correlationId, topic);
lock (syncProducerPoolForMetadataLock)
{
var brokerPartitionInfo = new BrokerPartitionInfo(syncProducerPoolForMetaData, tempTopicMetadatas,
tempTopicMetadatasLastUpdateTime, ProducerConfiguration.DefaultTopicMetaDataRefreshIntervalMS,
syncProducerPoolForMetaData.zkClient);
brokerPartitionInfo.UpdateInfo(versionId, correlationId, clientId, topic);
}
if (!tempTopicMetadatas.ContainsKey(topic))
{
throw new NoBrokerForTopicException(
string.Format(
"There is no metadata for topic {0}. Please check if all brokers of that topic live.", topic));
}
var metadata = tempTopicMetadatas[topic];
if (metadata.Error != ErrorMapping.NoError)
{
throw new KafkaException(
string.Format("The metadata status for topic {0} is abnormal, detail: ", topic), metadata.Error);
;
}
foreach (var p in metadata.PartitionsMetadata)
{
if (p.Leader != null && !partitionLeaders.ContainsKey(p.PartitionId))
{
partitionLeaders.Add(p.PartitionId, new Tuple <Broker, BrokerConfiguration>(
p.Leader,
new BrokerConfiguration
{
BrokerId = p.Leader.Id,
Host = p.Leader.Host,
Port = p.Leader.Port
}));
Logger.DebugFormat("RefreshMetadataInternal Topic {0} partition {1} has leader {2}", topic,
p.PartitionId, p.Leader.Id);
}
if (p.Leader == null)
{
Logger.ErrorFormat("RefreshMetadataInternal Topic {0} partition {1} does not have a leader yet.",
topic, p.PartitionId);
}
}
Logger.InfoFormat("RefreshMetadataInternal exit: {0} {1} {2} Topic:{3} ", versionId, clientId,
correlationId, topic);
}
public void Handle(IEnumerable <KeyedMessage <TKey, TValue> > events)
{
var serializedData = this.Serialize(events);
foreach (var keyed in serializedData)
{
var dataSize = keyed.Message.PayloadSize;
this.producerTopicStats.GetProducerTopicStats(keyed.Topic).ByteRate.Mark(dataSize);
this.producerTopicStats.GetProducerAllTopicsStats().ByteRate.Mark(dataSize);
}
var outstandingProduceRequests = serializedData;
var remainingRetries = this.Config.MessageSendMaxRetries + 1;
var correlationIdStart = this.correlationId.Get();
Logger.DebugFormat("Handling {0} events", events.Count());
while (remainingRetries > 0 && outstandingProduceRequests.Any())
{
foreach (var el in outstandingProduceRequests)
{
this.topicMetadataToRefresh.Add(el.Topic);
}
if (this.topicMetadataRefreshInterval >= TimeSpan.MinValue &&
(DateTime.Now - this.lastTopicMetadataRefeshTime) > this.topicMetadataRefreshInterval)
{
Util.SwallowError(
Logger,
() =>
this.brokerPartitionInfo.UpdateInfo(
new HashSet <string>(this.topicMetadataToRefresh), this.correlationId.GetAndIncrement()));
this.sendPartitionPerTopicCache.Clear();
this.topicMetadataToRefresh.Clear();
this.lastTopicMetadataRefeshTime = DateTime.Now;
}
outstandingProduceRequests = this.DispatchSerializedData(outstandingProduceRequests);
if (outstandingProduceRequests.Any())
{
Logger.InfoFormat("Back off for {0} ms before retrying send. Remaining retries = {1}", this.Config.RetryBackoffMs, remainingRetries - 1);
// back off and update the topic metadata cache before attempting another send operation
Thread.Sleep(this.Config.RetryBackoffMs);
Util.SwallowError(
Logger,
() =>
{
brokerPartitionInfo.UpdateInfo(
new HashSet <string>(outstandingProduceRequests.Select(r => r.Topic)), correlationId.GetAndIncrement());
sendPartitionPerTopicCache.Clear();
remainingRetries -= 1;
producerStats.ResendRate.Mark();
});
}
this.sendPartitionPerTopicCache.Clear();
remainingRetries -= 1;
}
if (outstandingProduceRequests.Any())
{
this.producerStats.FailedSendRate.Mark();
var correlationIdEnd = this.correlationId.Get();
Logger.ErrorFormat("Failed to send requests for topics {0} with correlation ids in [{1}, {2}]", string.Join(",", outstandingProduceRequests.Select(r => r.Topic)), correlationIdStart, correlationIdEnd);
throw new FailedToSendMessageException(
"Failed to send messages after " + this.Config.MessageSendMaxRetries + " tries");
}
}