|
public Get ( IKafkaConnection connections ) : Task |
||
| connections | IKafkaConnection | The server connections to query. Will cycle through the collection, starting at zero until a response is received. |
| return | Task |
|
public async Task ShouldBackoffRequestOnMultipleFailures(ErrorResponseCode errorCode)
{
var conn = Substitute.For<IKafkaConnection>();
conn.SendAsync(Arg.Any<IKafkaRequest<MetadataResponse>>())
.Returns(x => CreateMetadataResponse(errorCode),
x => CreateMetadataResponse(errorCode),
x => CreateMetadataResponse(errorCode),
x => CreateMetadataResponse(ErrorResponseCode.NoError));
using (var provider = new KafkaMetadataProvider(_log))
{
var response = await provider.Get(new[] { conn }, new[] { "Test" });
}
Received.InOrder(() =>
{
conn.SendAsync(Arg.Any<IKafkaRequest<MetadataResponse>>()).Wait();
_log.WarnFormat("Backing off metadata request retry. Waiting for {0}ms.", 100);
conn.SendAsync(Arg.Any<IKafkaRequest<MetadataResponse>>()).Wait();
_log.WarnFormat("Backing off metadata request retry. Waiting for {0}ms.", 400);
conn.SendAsync(Arg.Any<IKafkaRequest<MetadataResponse>>()).Wait();
_log.WarnFormat("Backing off metadata request retry. Waiting for {0}ms.", 900);
conn.SendAsync(Arg.Any<IKafkaRequest<MetadataResponse>>()).Wait();
});
}
/// <summary>
/// Refresh metadata Request will try to refresh only the topics that were expired in the
/// cache. If cacheExpiration is null: refresh metadata Request will try to refresh only
/// topics that are not in the cache.
/// </summary>
private async Task <bool> RefreshTopicMetadata(TimeSpan?cacheExpiration, TimeSpan timeout, params string[] topics)
{
try
{
await _taskLocker.WaitAsync(timeout).ConfigureAwait(false);
int missingFromCache = SearchCacheForTopics(topics, cacheExpiration).Missing.Count;
if (missingFromCache == 0)
{
return(false);
}
_kafkaOptions.Log.DebugFormat("BrokerRouter: Refreshing metadata for topics: {0}", string.Join(",", topics));
//get the connections to query against and get metadata
var connections = _defaultConnectionIndex.Values.Union(_brokerConnectionIndex.Values).ToArray();
var taskMetadata = _kafkaMetadataProvider.Get(connections, topics);
await Task.WhenAny(Task.Delay(timeout), taskMetadata).ConfigureAwait(false);
if (!taskMetadata.IsCompleted)
{
var ex = new Exception("Metadata refresh operation timed out");
throw ex;
}
var metadataResponse = await taskMetadata.ConfigureAwait(false);
UpdateInternalMetadataCache(metadataResponse);
}
finally
{
_taskLocker.Release();
}
return(true);
}
public void NewlyCreatedTopicShouldRetryUntilBrokerIsAssigned()
{
var expectedTopic = Guid.NewGuid().ToString();
var repo = new KafkaMetadataProvider(_options.Log);
var response = repo.Get(new[] { GetKafkaConnection() }, new[] { expectedTopic });
var topic = response.Topics.FirstOrDefault();
Assert.That(topic, Is.Not.Null);
Assert.That(topic.Name, Is.EqualTo(expectedTopic));
Assert.That(topic.ErrorCode, Is.EqualTo((int)ErrorResponseCode.NoError));
}
/// <summary>
/// Refresh metadata Request will try to refresh only the topics that were expired in the
/// cache. If cacheExpiration is null: refresh metadata Request will try to refresh only
/// topics that are not in the cache.
/// </summary>
private async Task <bool> RefreshTopicMetadata(TimeSpan?cacheExpiration, TimeSpan timeout, params string[] topics)
{
using (await _taskLocker.LockAsync().ConfigureAwait(false))
{
int missingFromCache = SearchCacheForTopics(topics, cacheExpiration).Missing.Count;
if (missingFromCache == 0)
{
return(false);
}
_kafkaOptions.Log.DebugFormat("BrokerRouter: Refreshing metadata for topics: {0}", string.Join(",", topics));
var connections = GetConnections();
var metadataRequestTask = _kafkaMetadataProvider.Get(connections, topics);
var metadataResponse = await metadataRequestTask.WithTimeout(timeout).ConfigureAwait(false);
UpdateInternalMetadataCache(metadataResponse);
}
return(true);
}
/// <summary>
/// Force a call to the kafka servers to refresh metadata for the given topics.
/// </summary>
/// <param name="topics">List of topics to update metadata for.</param>
/// <remarks>
/// This method will ignore the cache and initiate a call to the kafka servers for all given topics, updating the cache with the resulting metadata.
/// Only call this method to force a metadata update. For all other queries use <see cref="GetTopicMetadata"/> which uses cached values.
/// </remarks>
public void RefreshTopicMetadata(params string[] topics)
{
lock (_threadLock)
{
_kafkaOptions.Log.DebugFormat("BrokerRouter: Refreshing metadata for topics: {0}", string.Join(",", topics));
//get the connections to query against and get metadata
var connections = _defaultConnectionIndex.Values.Union(_brokerConnectionIndex.Values).ToArray();
var metadataResponse = _kafkaMetadataProvider.Get(connections, topics);
UpdateInternalMetadataCache(metadataResponse);
}
}
public void ShouldReturnWhenNoErrorReceived()
{
var conn = Substitute.For<IKafkaConnection>();
conn.SendAsync(Arg.Any<IKafkaRequest<MetadataResponse>>())
.Returns(x => CreateMetadataResponse(ErrorResponseCode.NoError));
using (var provider = new KafkaMetadataProvider(_log))
{
var response = provider.Get(new[] { conn }, new[] { "Test" });
}
conn.Received(1).SendAsync(Arg.Any<IKafkaRequest<MetadataResponse>>());
}
public async Task ShouldRetryWhenReceiveBrokerIdNegativeOne()
{
var conn = Substitute.For<IKafkaConnection>();
conn.SendAsync(Arg.Any<IKafkaRequest<MetadataResponse>>())
.Returns(x => CreateMetadataResponse(-1, "123", 1), x => CreateMetadataResponse(ErrorResponseCode.NoError));
using (var provider = new KafkaMetadataProvider(_log))
{
var response = await provider.Get(new[] { conn }, new[] { "Test" });
}
Received.InOrder(() =>
{
conn.SendAsync(Arg.Any<IKafkaRequest<MetadataResponse>>());
_log.WarnFormat("Backing off metadata request retry. Waiting for {0}ms.", 100);
conn.SendAsync(Arg.Any<IKafkaRequest<MetadataResponse>>());
});
}
public async Task ShouldThrowExceptionWhenNotARetriableErrorCode(ErrorResponseCode errorCode)
{
var conn = Substitute.For<IKafkaConnection>();
conn.SendAsync(Arg.Any<IKafkaRequest<MetadataResponse>>()).Returns(x => CreateMetadataResponse(errorCode));
using (var provider = new KafkaMetadataProvider(_log))
{
var response = await provider.Get(new[] { conn }, new[] { "Test" });
}
}
public async Task ShouldThrowExceptionWhenPortIsMissing(int port)
{
var conn = Substitute.For<IKafkaConnection>();
conn.SendAsync(Arg.Any<IKafkaRequest<MetadataResponse>>()).Returns(x => CreateMetadataResponse(1, "123", port));
using (var provider = new KafkaMetadataProvider(_log))
{
var response = await provider.Get(new[] { conn }, new[] { "Test" });
}
}
public void ShouldThrowExceptionWhenHostIsMissing(string host)
{
var conn = Substitute.For<IKafkaConnection>();
conn.SendAsync(Arg.Any<IKafkaRequest<MetadataResponse>>()).Returns(x => CreateMetadataResponse(1, host, 1));
using (var provider = new KafkaMetadataProvider(_log))
{
var response = provider.Get(new[] { conn }, new[] { "Test" });
}
}
