public Partition Select(Topic topic, byte[] key)
{
if (topic == null) throw new ArgumentNullException("topic");
if (topic.Partitions.Count <= 0) throw new ApplicationException(string.Format("Topic ({0}) has no partitions.", topic.Name));
//use round robin
var partitions = topic.Partitions;
if (key == null)
{
//use round robin
var paritionIndex = _roundRobinTracker.AddOrUpdate(topic.Name, p => 0, (s, i) =>
{
return ((i + 1) % partitions.Count);
});
return partitions[paritionIndex];
}
//use key hash
var partitionId = Crc32Provider.Compute(key) % partitions.Count;
var partition = partitions.FirstOrDefault(x => x.PartitionId == partitionId);
if (partition == null)
throw new InvalidPartitionException(string.Format("Hash function return partition id: {0}, but the available partitions are:{1}",
partitionId, string.Join(",", partitions.Select(x => x.PartitionId))));
return partition;
}
public Partition Select(Topic topic, string key)
{
if (topic == null) throw new ArgumentNullException("topic");
if (topic.Partitions.Count <= 0) throw new ApplicationException(string.Format("Topic ({0}) has no partitions.", topic.Name));
//use round robing
var partitions = topic.Partitions;
if (key == null)
{
return _roundRobinTracker.AddOrUpdate(topic.Name, x => partitions.First(), (s, i) =>
{
var index = partitions.FindIndex(0, p => p.Equals(i));
if (index == -1) return partitions.First();
if (++index >= partitions.Count) return partitions.First();
return partitions[index];
});
}
//use key hash
var partitionId = Math.Abs(key.GetHashCode()) % partitions.Count;
var partition = partitions.FirstOrDefault(x => x.PartitionId == partitionId);
if (partition == null)
throw new InvalidPartitionException(string.Format("Hash function return partition id: {0}, but the available partitions are:{1}",
partitionId, string.Join(",", partitions.Select(x => x.PartitionId))));
return partition;
}
public void KeyHashShouldThrowExceptionWhenChoosesAPartitionIdThatDoesNotExist()
{
var selector = new DefaultPartitionSelector();
var list = new List<Partition>(_topicA.Partitions);
list[1].PartitionId = 999;
var topic = new Topic
{
Name = "badPartition",
Partitions = list
};
selector.Select(topic, "1");
}
public static Topic FromStream(BigEndianBinaryReader stream)
{
var topic = new Topic
{
ErrorCode = stream.ReadInt16(),
Name = stream.ReadInt16String(),
Partitions = new List<Partition>()
};
var numPartitions = stream.ReadInt32();
for (int i = 0; i < numPartitions; i++)
{
topic.Partitions.Add(Partition.FromStream(stream));
}
return topic;
}
public void Setup()
{
_topicA = new Topic
{
Name = "a",
Partitions = new List<Partition>(new[]
{
new Partition
{
LeaderId = 0,
PartitionId = 0
},
new Partition
{
LeaderId = 1,
PartitionId = 1
}
})
};
_topicB = new Topic
{
Name = "b",
Partitions = new List<Partition>(new[]
{
new Partition
{
LeaderId = 0,
PartitionId = 0
},
new Partition
{
LeaderId = 1,
PartitionId = 1
}
})
};
}
private void EnsurePartitionPollingThreads()
{
try
{
if (Interlocked.Increment(ref _ensureOneThread) == 1)
{
_options.Log.DebugFormat("Consumer: Refreshing partitions for topic: {0}", _options.Topic);
var topic = _options.Router.GetTopicMetadata(_options.Topic);
if (topic.Count <= 0) throw new ApplicationException(string.Format("Unable to get metadata for topic:{0}.", _options.Topic));
_topic = topic.First();
//create one thread per partition, if they are in the white list.
foreach (var partition in _topic.Partitions)
{
var partitionId = partition.PartitionId;
if (_options.PartitionWhitelist.Count == 0 || _options.PartitionWhitelist.Any(x => x == partitionId))
{
_partitionPollingIndex.AddOrUpdate(partitionId,
i => ConsumeTopicPartitionAsync(_topic.Name, partitionId),
(i, task) => task);
}
}
}
}
catch (Exception ex)
{
_options.Log.ErrorFormat("Exception occured trying to setup consumer for topic:{0}. Exception={1}", _options.Topic, ex);
}
finally
{
Interlocked.Decrement(ref _ensureOneThread);
}
}
private BrokerRoute SelectConnectionFromCache(Topic topic, string key = null)
{
if (topic == null) throw new ArgumentNullException("topic");
var partition = _kafkaOptions.PartitionSelector.Select(topic, key);
return GetCachedRoute(topic.Name, partition);
}
private MetadataValidationResult ValidateTopic(Topic topic)
{
try
{
var errorCode = (ErrorResponseCode)topic.ErrorCode;
if (errorCode == ErrorResponseCode.NoError) return new MetadataValidationResult();
switch (errorCode)
{
case ErrorResponseCode.LeaderNotAvailable:
case ErrorResponseCode.OffsetsLoadInProgressCode:
case ErrorResponseCode.ConsumerCoordinatorNotAvailableCode:
return new MetadataValidationResult
{
Status = ValidationResult.Retry,
ErrorCode = errorCode,
Message = string.Format("Topic:{0} returned error code of {1}. Retrying.", topic.Name, errorCode)
};
}
return new MetadataValidationResult
{
Status = ValidationResult.Error,
ErrorCode = errorCode,
Exception = new InvalidTopicMetadataException(errorCode, "Topic:{0} returned an error of {1}.", topic.Name, errorCode)
};
}
catch
{
return new MetadataValidationResult
{
Status = ValidationResult.Error,
ErrorCode = ErrorResponseCode.Unknown,
Exception = new InvalidTopicMetadataException(ErrorResponseCode.Unknown, "Unknown error code returned in metadata response. ErrorCode: {0}", topic.ErrorCode)
};
}
}
private void RefreshTopicPartition()
{
try
{
if (Interlocked.Increment(ref _ensureOneThread) == 1)
{
var topic = _options.Router.GetTopicMetadata(_options.Topic);
if (topic.Count <= 0) throw new ApplicationException(string.Format("Unable to get metadata for topic:{0}.", _options.Topic));
_topic = topic.First();
//create one thread per partitions, if they are in the white list.
foreach (var partition in _topic.Partitions)
{
var partitionId = partition.PartitionId;
if (_options.PartitionWhitelist.Count == 0 || _options.PartitionWhitelist.Any(x => x == partitionId))
{
Func<int, PartitionConsumer> addValueFactory = _ =>
{
var partitionConsumer = new PartitionConsumer(
_options,
_topic.Name,
partitionId,
message => _fetchResponseQueue.Add(message));
partitionConsumer.Start();
return partitionConsumer;
};
this._partitionPollingIndex.AddOrUpdate(partitionId, addValueFactory, (i, consumer) => consumer);
}
}
}
}
catch (Exception ex)
{
_options.Log.ErrorFormat("Exception occured trying to setup consumer for topic:{0}. Exception={1}", _options.Topic, ex);
}
finally
{
Interlocked.Decrement(ref _ensureOneThread);
}
}
public void SelectorShouldThrowExceptionWhenPartitionsAreEmpty()
{
var selector = new DefaultPartitionSelector();
var topic = new Topic
{
Name = "emptyPartition",
Partitions = new List<Partition>()
};
selector.Select(topic, CreateKeyForPartition(1));
}
public void RoundRobinShouldEvenlyDistributeAcrossManyPartitions()
{
const int TotalPartitions = 100;
var selector = new DefaultPartitionSelector();
var partitions = new List<Partition>();
for (int i = 0; i < TotalPartitions; i++)
{
partitions.Add(new Partition { LeaderId = i, PartitionId = i });
}
var topic = new Topic { Name = "a", Partitions = partitions };
var bag = new ConcurrentBag<Partition>();
Parallel.For(0, TotalPartitions * 3, x => bag.Add(selector.Select(topic, null)));
var eachPartitionHasThree = bag.GroupBy(x => x.PartitionId).Count();
Assert.That(eachPartitionHasThree, Is.EqualTo(TotalPartitions), "Each partition should have received three selections.");
}