public Partition Select(Topic topic, byte[] key)
        {
            if (topic == null) throw new ArgumentNullException("topic");
            if (topic.Partitions.Count <= 0) throw new ApplicationException(string.Format("Topic ({0}) has no partitions.", topic.Name));

            //use round robin
            var partitions = topic.Partitions;
            if (key == null)
            {
                //use round robin
                var paritionIndex = _roundRobinTracker.AddOrUpdate(topic.Name, p => 0, (s, i) =>
                    {
                        return ((i + 1) % partitions.Count);
                    });

                return partitions[paritionIndex];
            }

            //use key hash
            var partitionId = Crc32Provider.Compute(key) % partitions.Count;
            var partition = partitions.FirstOrDefault(x => x.PartitionId == partitionId);

            if (partition == null)
                throw new InvalidPartitionException(string.Format("Hash function return partition id: {0}, but the available partitions are:{1}",
                                                                            partitionId, string.Join(",", partitions.Select(x => x.PartitionId))));

            return partition;
        }
        public Partition Select(Topic topic, string key)
        {
            if (topic == null) throw new ArgumentNullException("topic");
            if (topic.Partitions.Count <= 0) throw new ApplicationException(string.Format("Topic ({0}) has no partitions.", topic.Name));

            //use round robing
            var partitions = topic.Partitions;
            if (key == null)
            {
                return _roundRobinTracker.AddOrUpdate(topic.Name, x => partitions.First(), (s, i) =>
                    {
                        var index = partitions.FindIndex(0, p => p.Equals(i));
                        if (index == -1) return partitions.First();
                        if (++index >= partitions.Count) return partitions.First();
                        return partitions[index];
                    });
            }

            //use key hash
            var partitionId = Math.Abs(key.GetHashCode()) % partitions.Count;
            var partition = partitions.FirstOrDefault(x => x.PartitionId == partitionId);

            if (partition == null)
                throw new InvalidPartitionException(string.Format("Hash function return partition id: {0}, but the available partitions are:{1}",
                                                                            partitionId, string.Join(",", partitions.Select(x => x.PartitionId))));

            return partition;
        }
        public void KeyHashShouldThrowExceptionWhenChoosesAPartitionIdThatDoesNotExist()
        {
            var selector = new DefaultPartitionSelector();
            var list = new List<Partition>(_topicA.Partitions);
            list[1].PartitionId = 999;
            var topic = new Topic
                {
                    Name = "badPartition",
                    Partitions = list
                };

            selector.Select(topic, "1");
        }
Exemplo n.º 4
0
        public static Topic FromStream(BigEndianBinaryReader stream)
        {
            var topic = new Topic
                {
                    ErrorCode = stream.ReadInt16(),
                    Name = stream.ReadInt16String(),
                    Partitions = new List<Partition>()
                };

            var numPartitions = stream.ReadInt32();
            for (int i = 0; i < numPartitions; i++)
            {
                topic.Partitions.Add(Partition.FromStream(stream));
            }

            return topic;
        }
        public void Setup()
        {
            _topicA = new Topic
            {
                Name = "a",
                Partitions = new List<Partition>(new[]
                {
                    new Partition
                        {
                            LeaderId = 0,
                            PartitionId = 0
                        },
                    new Partition
                        {
                            LeaderId = 1,
                            PartitionId = 1
                        }
                })
            };

            _topicB = new Topic
            {
                Name = "b",
                Partitions = new List<Partition>(new[]
                {
                    new Partition
                        {
                            LeaderId = 0,
                            PartitionId = 0
                        },
                    new Partition
                        {
                            LeaderId = 1,
                            PartitionId = 1
                        }
                })
            };
        }
Exemplo n.º 6
0
        private void EnsurePartitionPollingThreads()
        {
            try
            {
                if (Interlocked.Increment(ref _ensureOneThread) == 1)
                {
                    _options.Log.DebugFormat("Consumer: Refreshing partitions for topic: {0}", _options.Topic);
                    var topic = _options.Router.GetTopicMetadata(_options.Topic);
                    if (topic.Count <= 0) throw new ApplicationException(string.Format("Unable to get metadata for topic:{0}.", _options.Topic));
                    _topic = topic.First();

                    //create one thread per partition, if they are in the white list.
                    foreach (var partition in _topic.Partitions)
                    {
                        var partitionId = partition.PartitionId;
                        if (_options.PartitionWhitelist.Count == 0 || _options.PartitionWhitelist.Any(x => x == partitionId))
                        {
                            _partitionPollingIndex.AddOrUpdate(partitionId,
                                                               i => ConsumeTopicPartitionAsync(_topic.Name, partitionId),
                                                               (i, task) => task);
                        }
                    }
                }
            }
            catch (Exception ex)
            {
                _options.Log.ErrorFormat("Exception occured trying to setup consumer for topic:{0}.  Exception={1}", _options.Topic, ex);
            }
            finally
            {
                Interlocked.Decrement(ref _ensureOneThread);
            }
        }
Exemplo n.º 7
0
 private BrokerRoute SelectConnectionFromCache(Topic topic, string key = null)
 {
     if (topic == null) throw new ArgumentNullException("topic");
     var partition = _kafkaOptions.PartitionSelector.Select(topic, key);
     return GetCachedRoute(topic.Name, partition);
 }
Exemplo n.º 8
0
        private MetadataValidationResult ValidateTopic(Topic topic)
        {
            try
            {
                var errorCode = (ErrorResponseCode)topic.ErrorCode;

                if (errorCode == ErrorResponseCode.NoError) return new MetadataValidationResult();

                switch (errorCode)
                {
                    case ErrorResponseCode.LeaderNotAvailable:
                    case ErrorResponseCode.OffsetsLoadInProgressCode:
                    case ErrorResponseCode.ConsumerCoordinatorNotAvailableCode:
                        return new MetadataValidationResult
                        {
                            Status = ValidationResult.Retry,
                            ErrorCode = errorCode,
                            Message = string.Format("Topic:{0} returned error code of {1}.  Retrying.", topic.Name, errorCode)
                        };
                }

                return new MetadataValidationResult
                {
                    Status = ValidationResult.Error,
                    ErrorCode = errorCode,
                    Exception = new InvalidTopicMetadataException(errorCode, "Topic:{0} returned an error of {1}.", topic.Name, errorCode)
                };
            }
            catch
            {
                return new MetadataValidationResult
                {
                    Status = ValidationResult.Error,
                    ErrorCode = ErrorResponseCode.Unknown,
                    Exception = new InvalidTopicMetadataException(ErrorResponseCode.Unknown, "Unknown error code returned in metadata response.  ErrorCode: {0}", topic.ErrorCode)
                };
            }
        }
Exemplo n.º 9
0
        private void RefreshTopicPartition()
        {
            try
            {
                if (Interlocked.Increment(ref _ensureOneThread) == 1)
                {
                    var topic = _options.Router.GetTopicMetadata(_options.Topic);
                    if (topic.Count <= 0) throw new ApplicationException(string.Format("Unable to get metadata for topic:{0}.", _options.Topic));
                    _topic = topic.First();

                    //create one thread per partitions, if they are in the white list.
                    foreach (var partition in _topic.Partitions)
                    {
                        var partitionId = partition.PartitionId;
                        if (_options.PartitionWhitelist.Count == 0 || _options.PartitionWhitelist.Any(x => x == partitionId))
                        {
                            Func<int, PartitionConsumer> addValueFactory = _ =>
                                {
                                    var partitionConsumer = new PartitionConsumer(
                                        _options,
                                        _topic.Name,
                                        partitionId,
                                        message => _fetchResponseQueue.Add(message));
                                    partitionConsumer.Start();
                                    return partitionConsumer;
                                };

                            this._partitionPollingIndex.AddOrUpdate(partitionId, addValueFactory, (i, consumer) => consumer);
                        }
                    }
                }
            }
            catch (Exception ex)
            {
                _options.Log.ErrorFormat("Exception occured trying to setup consumer for topic:{0}.  Exception={1}", _options.Topic, ex);
            }
            finally
            {
                Interlocked.Decrement(ref _ensureOneThread);
            }
        }
 public void SelectorShouldThrowExceptionWhenPartitionsAreEmpty()
 {
     var selector = new DefaultPartitionSelector();
     var topic = new Topic
     {
         Name = "emptyPartition",
         Partitions = new List<Partition>()
     };
     selector.Select(topic, CreateKeyForPartition(1));
 }
        public void RoundRobinShouldEvenlyDistributeAcrossManyPartitions()
        {
            const int TotalPartitions = 100;
            var selector = new DefaultPartitionSelector();
            var partitions = new List<Partition>();
            for (int i = 0; i < TotalPartitions; i++)
            {
                partitions.Add(new Partition { LeaderId = i, PartitionId = i });
            }
            var topic = new Topic { Name = "a", Partitions = partitions };

            var bag = new ConcurrentBag<Partition>();
            Parallel.For(0, TotalPartitions * 3, x => bag.Add(selector.Select(topic, null)));

            var eachPartitionHasThree = bag.GroupBy(x => x.PartitionId).Count();
                
            Assert.That(eachPartitionHasThree, Is.EqualTo(TotalPartitions), "Each partition should have received three selections.");
        }