private void UpdateInternalMetadataCache(MetadataResponse metadata) { //resolve each broker var brokerEndpoints = metadata.Brokers.Select(broker => new { Broker = broker, Endpoint = _kafkaOptions.KafkaConnectionFactory.Resolve(broker.Address, _kafkaOptions.Log) }); foreach (var broker in brokerEndpoints) { //if the connection is in our default connection index already, remove it and assign it to the broker index. IKafkaConnection connection; if (_defaultConnectionIndex.TryRemove(broker.Endpoint, out connection)) { UpsertConnectionToBrokerConnectionIndex(broker.Broker.BrokerId, connection); } else { connection = _kafkaOptions.KafkaConnectionFactory.Create(broker.Endpoint, _kafkaOptions.ResponseTimeoutMs, _kafkaOptions.Log); UpsertConnectionToBrokerConnectionIndex(broker.Broker.BrokerId, connection); } } foreach (var topic in metadata.Topics) { var localTopic = topic; _topicIndex.AddOrUpdate(topic.Name, s => localTopic, (s, existing) => localTopic); } }
private IEnumerable <MetadataValidationResult> ValidateResponse(MetadataResponse metadata) { foreach (var broker in metadata.Brokers) { yield return(ValidateBroker(broker)); } foreach (var topic in metadata.Topics) { yield return(ValidateTopic(topic)); } }
/// <summary> /// Given a collection of server connections, query for the topic metadata. /// </summary> /// <param name="connections">The server connections to query. Will cycle through the collection, starting at zero until a response is received.</param> /// <param name="topics">The collection of topics to get metadata for.</param> /// <returns>MetadataResponse validated to be complete.</returns> public MetadataResponse Get(IKafkaConnection[] connections, IEnumerable <string> topics) { var request = new MetadataRequest { Topics = topics.ToList() }; if (request.Topics.Count <= 0) { return(null); } var performRetry = false; var retryAttempt = 0; MetadataResponse metadataResponse = null; do { performRetry = false; metadataResponse = GetMetadataResponse(connections, request); if (metadataResponse == null) { return(null); } foreach (var validation in ValidateResponse(metadataResponse)) { switch (validation.Status) { case ValidationResult.Retry: performRetry = true; _log.WarnFormat(validation.Message); break; case ValidationResult.Error: throw validation.Exception; } } BackoffOnRetry(++retryAttempt, performRetry); } while (_interrupted == false && performRetry); return(metadataResponse); }