private void UpdateInternalMetadataCache(MetadataResponse metadata)
{
//resolve each broker
var brokerEndpoints = metadata.Brokers.Select(broker => new
{
Broker = broker,
Endpoint = _kafkaOptions.KafkaConnectionFactory.Resolve(broker.Address, _kafkaOptions.Log)
});
foreach (var broker in brokerEndpoints)
{
//if the connection is in our default connection index already, remove it and assign it to the broker index.
IKafkaConnection connection;
if (_defaultConnectionIndex.TryRemove(broker.Endpoint, out connection))
{
UpsertConnectionToBrokerConnectionIndex(broker.Broker.BrokerId, connection);
}
else
{
connection = _kafkaOptions.KafkaConnectionFactory.Create(broker.Endpoint, _kafkaOptions.ResponseTimeoutMs, _kafkaOptions.Log);
UpsertConnectionToBrokerConnectionIndex(broker.Broker.BrokerId, connection);
}
}
foreach (var topic in metadata.Topics)
{
var localTopic = topic;
_topicIndex.AddOrUpdate(topic.Name, s => localTopic, (s, existing) => localTopic);
}
}
private IEnumerable <MetadataValidationResult> ValidateResponse(MetadataResponse metadata)
{
foreach (var broker in metadata.Brokers)
{
yield return(ValidateBroker(broker));
}
foreach (var topic in metadata.Topics)
{
yield return(ValidateTopic(topic));
}
}
/// <summary>
/// Given a collection of server connections, query for the topic metadata.
/// </summary>
/// <param name="connections">The server connections to query. Will cycle through the collection, starting at zero until a response is received.</param>
/// <param name="topics">The collection of topics to get metadata for.</param>
/// <returns>MetadataResponse validated to be complete.</returns>
public MetadataResponse Get(IKafkaConnection[] connections, IEnumerable <string> topics)
{
var request = new MetadataRequest {
Topics = topics.ToList()
};
if (request.Topics.Count <= 0)
{
return(null);
}
var performRetry = false;
var retryAttempt = 0;
MetadataResponse metadataResponse = null;
do
{
performRetry = false;
metadataResponse = GetMetadataResponse(connections, request);
if (metadataResponse == null)
{
return(null);
}
foreach (var validation in ValidateResponse(metadataResponse))
{
switch (validation.Status)
{
case ValidationResult.Retry:
performRetry = true;
_log.WarnFormat(validation.Message);
break;
case ValidationResult.Error:
throw validation.Exception;
}
}
BackoffOnRetry(++retryAttempt, performRetry);
} while (_interrupted == false && performRetry);
return(metadataResponse);
}
