private async Task<MetadataResponse> Get(IKafkaConnection[] connections, MetadataRequest request)
{
var maxRetryAttempt = 2;
var performRetry = false;
var retryAttempt = 0;
MetadataResponse metadataResponse = null;
do
{
performRetry = false;
metadataResponse = await GetMetadataResponse(connections, request);
if (metadataResponse == null) return null;
foreach (var validation in ValidateResponse(metadataResponse))
{
switch (validation.Status)
{
case ValidationResult.Retry:
performRetry = true;
_log.WarnFormat(validation.Message);
break;
case ValidationResult.Error:
throw validation.Exception;
}
}
await BackoffOnRetry(++retryAttempt, performRetry).ConfigureAwait(false);
} while (retryAttempt < maxRetryAttempt && _interrupted == false && performRetry);
return metadataResponse;
}
/// <summary>
/// Given a collection of server connections, query for the topic metadata.
/// </summary>
/// <param name="connections">The server connections to query. Will cycle through the collection, starting at zero until a response is received.</param>
/// <param name="topics">The collection of topics to get metadata for.</param>
/// <returns>MetadataResponse validated to be complete.</returns>
public MetadataResponse Get(IKafkaConnection[] connections, IEnumerable<string> topics)
{
var request = new MetadataRequest { Topics = topics.ToList() };
if (request.Topics.Count <= 0) return null;
var performRetry = false;
var retryAttempt = 0;
MetadataResponse metadataResponse = null;
do
{
performRetry = false;
metadataResponse = GetMetadataResponse(connections, request);
if (metadataResponse == null) return null;
foreach (var validation in ValidateResponse(metadataResponse))
{
switch (validation.Status)
{
case ValidationResult.Retry:
performRetry = true;
_log.WarnFormat(validation.Message);
break;
case ValidationResult.Error:
throw validation.Exception;
}
}
BackoffOnRetry(++retryAttempt, performRetry);
} while (_interrupted == false && performRetry);
return metadataResponse;
}
private void UpsertConnectionToBrokerConnectionIndex(int brokerId, IKafkaConnection newConnection)
{
//associate the connection with the broker id, and add or update the reference
_brokerConnectionIndex.AddOrUpdate(brokerId,
i => newConnection,
(i, existingConnection) =>
{
//if a connection changes for a broker close old connection and create a new one
if (existingConnection.Endpoint.Equals(newConnection.Endpoint))
{
return(existingConnection);
}
_kafkaOptions.Log.WarnFormat("Broker:{0} Uri changed from:{1} to {2}", brokerId, existingConnection.Endpoint, newConnection.Endpoint);
using (existingConnection)
{
return(newConnection);
}
});
}
public SyncProducer(SyncProducerConfiguration config, IKafkaConnection connection)
{
Guard.NotNull(config, "config");
this.Config = config;
this.connection = connection;
}
private MetadataResponse GetMetadataResponse(IKafkaConnection[] connections, MetadataRequest request)
{
//try each default broker until we find one that is available
foreach (var conn in connections)
{
try
{
var response = conn.SendAsync(request).Result;
if (response != null && response.Count > 0)
{
return response.FirstOrDefault();
}
}
catch (Exception ex)
{
_log.WarnFormat("Failed to contact Kafka server={0}. Trying next default server. Exception={1}", conn.Endpoint, ex);
}
}
throw new ServerUnreachableException(
"Unable to query for metadata from any of the default Kafka servers. At least one provided server must be available. Server list: {0}",
string.Join(", ", connections.Select(x => x.ToString())));
}
private void UpsertConnectionToBrokerConnectionIndex(int brokerId, IKafkaConnection newConnection)
{
//associate the connection with the broker id, and add or update the reference
_brokerConnectionIndex.AddOrUpdate(brokerId,
i => newConnection,
(i, existingConnection) =>
{
//if a connection changes for a broker close old connection and create a new one
if (existingConnection.Endpoint.Equals(newConnection.Endpoint)) return existingConnection;
_kafkaOptions.Log.WarnFormat("Broker:{0} Uri changed from:{1} to {2}", brokerId, existingConnection.Endpoint, newConnection.Endpoint);
using (existingConnection)
{
return newConnection;
}
});
}
public SyncProducer(SyncProducerConfiguration config, IKafkaConnection connection)
{
Guard.NotNull(config, "config");
this.Config = config;
this.connection = connection;
}
/// <summary>
/// Given a collection of server connections, query for all topics metadata.
/// </summary>
/// <param name="connections">The server connections to query. Will cycle through the collection, starting at zero until a response is received.</param>
/// <returns>MetadataResponse validated to be complete.</returns>
public Task<MetadataResponse> Get(IKafkaConnection[] connections)
{
var request = new MetadataRequest();
return Get(connections, request);
}
/// <summary>
/// Given a collection of server connections, query for the topic metadata.
/// </summary>
/// <param name="connections">The server connections to query. Will cycle through the collection, starting at zero until a response is received.</param>
/// <param name="topics">The collection of topics to get metadata for.</param>
/// <returns>MetadataResponse validated to be complete.</returns>
public Task<MetadataResponse> Get(IKafkaConnection[] connections, IEnumerable<string> topics)
{
var request = new MetadataRequest { Topics = topics.ToList() };
if (request.Topics.Count <= 0) return null;
return Get(connections, request);
}
