public Partition Select(Topic topic, byte[] key)
{
if (topic == null) throw new ArgumentNullException("topic");
if (topic.Partitions.Count <= 0) throw new ApplicationException(string.Format("Topic ({0}) has no partitions.", topic.Name));
//use round robin
var partitions = topic.Partitions;
if (key == null)
{
//use round robin
var paritionIndex = _roundRobinTracker.AddOrUpdate(topic.Name, p => 0, (s, i) =>
{
return ((i + 1) % partitions.Count);
});
return partitions[paritionIndex];
}
//use key hash
var partitionId = BinaryCrc32.Compute(key) % partitions.Count;
var partition = partitions.FirstOrDefault(x => x.PartitionId == partitionId);
if (partition == null)
throw new InvalidPartitionException(string.Format("Hash function return partition id: {0}, but the available partitions are:{1}",
partitionId, string.Join(",", partitions.Select(x => x.PartitionId))));
return partition;
}
public static Topic FromStream(BinaryReader stream)
{
var topic = new Topic
{
ErrorCode = stream.ReadInt16(),
Name = stream.ReadInt16String(),
Partitions = new List<Partition>()
};
var numPartitions = stream.ReadInt32();
for (int i = 0; i < numPartitions; i++)
{
topic.Partitions.Add(Partition.FromStream(stream));
}
return topic;
}
private MetadataValidationResult ValidateTopic(Topic topic)
{
try
{
var errorCode = (ErrorResponseCode)topic.ErrorCode;
if (errorCode == ErrorResponseCode.NoError) return new MetadataValidationResult();
switch (errorCode)
{
case ErrorResponseCode.LeaderNotAvailable:
case ErrorResponseCode.OffsetsLoadInProgressCode:
case ErrorResponseCode.ConsumerCoordinatorNotAvailableCode:
return new MetadataValidationResult
{
Status = ValidationResult.Retry,
ErrorCode = errorCode,
Message = string.Format("Topic:{0} returned error code of {1}. Retrying.", topic.Name, errorCode)
};
}
return new MetadataValidationResult
{
Status = ValidationResult.Error,
ErrorCode = errorCode,
Exception = new InvalidTopicMetadataException(errorCode, "Topic:{0} returned an error of {1}.", topic.Name, errorCode)
};
}
catch
{
return new MetadataValidationResult
{
Status = ValidationResult.Error,
ErrorCode = ErrorResponseCode.Unknown,
Exception = new InvalidTopicMetadataException(ErrorResponseCode.Unknown, "Unknown error code returned in metadata response. ErrorCode: {0}", topic.ErrorCode)
};
}
}
private void RefreshTopicPartitions()
{
try
{
if (Interlocked.Increment(ref _ensureOneThread) == 1)
{
_options.Log.DebugFormat("Consumer: Refreshing partitions for topic: {0}", _options.Topic);
var topic = _options.Router.GetTopicMetadata(_options.Topic);
if (topic.Count <= 0) throw new ApplicationException(string.Format("Unable to get metadata for topic:{0}.", _options.Topic));
_topic = topic.First();
//create one thread per partition, if they are in the white list.
foreach (var partition in _topic.Partitions)
{
var partitionId = partition.PartitionId;
if (_options.PartitionWhitelist.Count == 0 || _options.PartitionWhitelist.Any(x => x == partitionId))
{
_partitionPollingIndex.AddOrUpdate(partitionId,
i => ConsumeTopicPartitionAsync(_topic.Name, partitionId),
(i, task) => task);
}
}
}
}
catch (Exception ex)
{
_options.Log.ErrorFormat("Exception occured trying to setup consumer for topic:{0}. Exception={1}", _options.Topic, ex);
}
finally
{
Interlocked.Decrement(ref _ensureOneThread);
}
}