public Partition Select(Topic topic, byte[] key)
        {
            if (topic == null) throw new ArgumentNullException("topic");
            if (topic.Partitions.Count <= 0) throw new ApplicationException(string.Format("Topic ({0}) has no partitions.", topic.Name));

            //use round robin
            var partitions = topic.Partitions;
            if (key == null)
            {
                //use round robin
                var paritionIndex = _roundRobinTracker.AddOrUpdate(topic.Name, p => 0, (s, i) =>
                    {
                        return ((i + 1) % partitions.Count);
                    });

                return partitions[paritionIndex];
            }

            //use key hash
            var partitionId = BinaryCrc32.Compute(key) % partitions.Count;
            var partition = partitions.FirstOrDefault(x => x.PartitionId == partitionId);

            if (partition == null)
                throw new InvalidPartitionException(string.Format("Hash function return partition id: {0}, but the available partitions are:{1}",
                                                                            partitionId, string.Join(",", partitions.Select(x => x.PartitionId))));

            return partition;
        }
Ejemplo n.º 2
0
        public static Topic FromStream(BinaryReader stream)
        {
            var topic = new Topic
                {
                    ErrorCode = stream.ReadInt16(),
                    Name = stream.ReadInt16String(),
                    Partitions = new List<Partition>()
                };

            var numPartitions = stream.ReadInt32();
            for (int i = 0; i < numPartitions; i++)
            {
                topic.Partitions.Add(Partition.FromStream(stream));
            }

            return topic;
        }
        private MetadataValidationResult ValidateTopic(Topic topic)
        {
            try
            {
                var errorCode = (ErrorResponseCode)topic.ErrorCode;

                if (errorCode == ErrorResponseCode.NoError) return new MetadataValidationResult();

                switch (errorCode)
                {
                    case ErrorResponseCode.LeaderNotAvailable:
                    case ErrorResponseCode.OffsetsLoadInProgressCode:
                    case ErrorResponseCode.ConsumerCoordinatorNotAvailableCode:
                        return new MetadataValidationResult
                        {
                            Status = ValidationResult.Retry,
                            ErrorCode = errorCode,
                            Message = string.Format("Topic:{0} returned error code of {1}.  Retrying.", topic.Name, errorCode)
                        };
                }

                return new MetadataValidationResult
                {
                    Status = ValidationResult.Error,
                    ErrorCode = errorCode,
                    Exception = new InvalidTopicMetadataException(errorCode, "Topic:{0} returned an error of {1}.", topic.Name, errorCode)
                };
            }
            catch
            {
                return new MetadataValidationResult
                {
                    Status = ValidationResult.Error,
                    ErrorCode = ErrorResponseCode.Unknown,
                    Exception = new InvalidTopicMetadataException(ErrorResponseCode.Unknown, "Unknown error code returned in metadata response.  ErrorCode: {0}", topic.ErrorCode)
                };
            }
        }
Ejemplo n.º 4
0
        private void RefreshTopicPartitions()
        {
            try
            {
                if (Interlocked.Increment(ref _ensureOneThread) == 1)
                {
                    _options.Log.DebugFormat("Consumer: Refreshing partitions for topic: {0}", _options.Topic);
                    var topic = _options.Router.GetTopicMetadata(_options.Topic);
                    if (topic.Count <= 0) throw new ApplicationException(string.Format("Unable to get metadata for topic:{0}.", _options.Topic));
                    _topic = topic.First();

                    //create one thread per partition, if they are in the white list.
                    foreach (var partition in _topic.Partitions)
                    {
                        var partitionId = partition.PartitionId;
                        if (_options.PartitionWhitelist.Count == 0 || _options.PartitionWhitelist.Any(x => x == partitionId))
                        {
                            _partitionPollingIndex.AddOrUpdate(partitionId,
                                                               i => ConsumeTopicPartitionAsync(_topic.Name, partitionId),
                                                               (i, task) => task);
                        }
                    }
                }
            }
            catch (Exception ex)
            {
                _options.Log.ErrorFormat("Exception occured trying to setup consumer for topic:{0}.  Exception={1}", _options.Topic, ex);
            }
            finally
            {
                Interlocked.Decrement(ref _ensureOneThread);
            }
        }