public MetadataResponse.Partition Select(MetadataResponse.Topic topic, ArraySegment <byte> key)
{
if (topic == null)
{
throw new ArgumentNullException(nameof(topic));
}
if (topic.partition_metadata.Count == 0)
{
throw new RoutingException($"No partitions to choose on {topic}.");
}
if (key.Count == 0)
{
return(RoundRobinPartitionSelector.Singleton.Select(topic, key));
}
// use key hash
var partitionId = Crc32.Compute(key) % topic.partition_metadata.Count;
var partition = topic.partition_metadata.FirstOrDefault(x => x.partition_id == partitionId);
if (partition != null)
{
return(partition);
}
throw new RoutingException($"Hash function return partition {partitionId}, but the available partitions are {string.Join(",", topic.partition_metadata.Select(x => x.partition_id))}");
}
private void EnsurePartitionPollingThreads()
{
try
{
if (Interlocked.Increment(ref _ensureOneThread) == 1)
{
_options.Log.Debug(() => LogEvent.Create($"Consumer: Refreshing partitions for topic/{_options.Topic}"));
_topic = _options.Router.GetTopicMetadataAsync(_options.Topic, CancellationToken.None).Result;
//create one thread per partition, if they are in the white list.
foreach (var partition in _topic.Partitions)
{
var partitionId = partition.PartitionId;
if (_options.PartitionWhitelist.Count == 0 || _options.PartitionWhitelist.Any(x => x == partitionId))
{
_partitionPollingIndex.AddOrUpdate(partitionId,
i => ConsumeTopicPartitionAsync(_topic.TopicName, partitionId, CancellationToken.None),
(i, task) => task);
}
}
}
}
catch (Exception ex)
{
_options.Log.Error(LogEvent.Create(ex, $"Trying to setup consumer for topic/{_options.Topic}"));
}
finally
{
Interlocked.Decrement(ref _ensureOneThread);
}
}
public void SelectorShouldThrowExceptionWhenPartitionsAreEmpty()
{
var selector = new PartitionSelector();
var topic = new MetadataResponse.Topic("emptyPartition");
Assert.Throws <RoutingException>(() => selector.Select(topic, CreateKeyForPartition(1)));
}
public void PartitionSelectionOnEmptyKeyHashShouldNotFail()
{
var selector = new PartitionSelector();
var topic = new MetadataResponse.Topic("badPartition", partitions: new [] {
new MetadataResponse.Partition(0, 0),
new MetadataResponse.Partition(999, 1)
});
Assert.That(selector.Select(topic, new ArraySegment <byte>()), Is.Not.Null);
}
public void KeyHashShouldThrowExceptionWhenChoosesAPartitionIdThatDoesNotExist()
{
var selector = new PartitionSelector();
var topic = new MetadataResponse.Topic("badPartition", partitions: new [] {
new MetadataResponse.Partition(0, 0),
new MetadataResponse.Partition(999, 1)
});
Assert.Throws <RoutingException>(() => selector.Select(topic, CreateKeyForPartition(1)));
}
public void Setup()
{
_topicA = new MetadataResponse.Topic("a", ErrorCode.NONE, new [] {
new MetadataResponse.Partition(0, 0),
new MetadataResponse.Partition(1, 1),
});
_topicB = new MetadataResponse.Topic("b", ErrorCode.NONE, new [] {
new MetadataResponse.Partition(0, 0),
new MetadataResponse.Partition(1, 1),
});
}
private static MetadataResult ValidateTopic(MetadataResponse.Topic topic)
{
var errorCode = topic.ErrorCode;
if (errorCode == ErrorResponseCode.None)
{
return(new MetadataResult(isValid: true));
}
if (errorCode.IsRetryable())
{
return(new MetadataResult(errorCode, null, $"topic/{topic.TopicName} returned error code of {errorCode}: Retrying"));
}
return(new MetadataResult(errorCode, false, $"topic/{topic.TopicName} returned an error of {errorCode}"));
}
public MetadataResponse.Partition Select(MetadataResponse.Topic topic, ArraySegment <byte> key)
{
if (topic == null)
{
throw new ArgumentNullException(nameof(topic));
}
if (topic.partition_metadata.Count == 0)
{
throw new RoutingException($"No partitions to choose on {topic}.");
}
var paritionIndex = _tracker.AddOrUpdate(topic.topic, p => 0, (s, i) => (i + 1) % topic.partition_metadata.Count);
return(topic.partition_metadata[paritionIndex]);
}
private BrokerRoute GetBrokerRoute(string topicName, int partitionId, MetadataResponse.Topic topic)
{
var partition = topic.Partitions.FirstOrDefault(x => x.PartitionId == partitionId);
if (partition == null)
{
throw new CachedMetadataException($"The topic ({topicName}) has no partitionId {partitionId} defined.")
{
TopicName = topicName,
Partition = partitionId
}
}
;
return(GetCachedRoute(topicName, partition));
}
public MetadataResponse.Partition Select(MetadataResponse.Topic topic, byte[] key)
{
if (topic == null)
{
throw new ArgumentNullException(nameof(topic));
}
if (topic.Partitions.Count <= 0)
{
throw new CachedMetadataException($"topic/{topic.TopicName} has no partitions.")
{
TopicName = topic.TopicName
}
}
;
long partitionId;
var partitions = topic.Partitions;
if (key == null)
{
// use round robin
var paritionIndex = _roundRobinTracker.AddOrUpdate(topic.TopicName, p => 0, (s, i) => (i + 1) % partitions.Count);
return(partitions[paritionIndex]);
}
else
{
// use key hash
partitionId = Crc32Provider.Compute(key) % partitions.Count;
var partition = partitions.FirstOrDefault(x => x.PartitionId == partitionId);
if (partition != null)
{
return(partition);
}
}
throw new CachedMetadataException($"Hash function return partition/{partitionId}, but the available partitions are {string.Join(",", partitions.Select(x => x.PartitionId))}")
{
TopicName = topic.TopicName,
Partition = (int)partitionId
};
}
}
public void RoundRobinShouldEvenlyDistributeAcrossManyPartitions()
{
RoundRobinPartitionSelector.Singleton.Reset();
const int TotalPartitions = 100;
var partitions = new List <MetadataResponse.Partition>();
for (int i = 0; i < TotalPartitions; i++)
{
partitions.Add(new MetadataResponse.Partition(i, i));
}
var topic = new MetadataResponse.Topic("a", partitions: partitions);
var bag = new ConcurrentBag <MetadataResponse.Partition>();
Parallel.For(0, TotalPartitions * 3, x => bag.Add(RoundRobinPartitionSelector.Singleton.Select(topic, new ArraySegment <byte>())));
var eachPartitionHasThree = bag.GroupBy(x => x.partition_id).Count();
Assert.That(eachPartitionHasThree, Is.EqualTo(TotalPartitions), "Each partition should have received three selections.");
}
public void RoundRobinShouldThrowExceptionWhenPartitionsAreEmpty()
{
var topic = new MetadataResponse.Topic("emptyPartition");
Assert.Throws <RoutingException>(() => RoundRobinPartitionSelector.Singleton.Select(topic, CreateKeyForPartition(1)));
}
private TopicConnection GetCachedTopicConnection(string topicName, int partitionId, MetadataResponse.Topic topic)
{
var partition = topic.partition_metadata.FirstOrDefault(x => x.partition_id == partitionId);
if (partition == null)
{
throw new RoutingException($"The topic ({topicName}) has no partitionId {partitionId} defined.");
}
return(GetCachedTopicConnection(topicName, partition));
}
