Beispiel #1
0
 private async Task Offset(string topic, int partition, long time)
 {
     await NodeFetch(topic, partition,
                     l =>
                     _offsetBatchStrategy.Send(l,
                                               (new OffsetMessage {
         Topic = topic, Partition = partition, MaxNumberOfOffsets = 1, Time = time
     })));
 }
Beispiel #2
0
 private async Task Fetch(string topic, int partition, long offset)
 {
     await NodeFetch(topic, partition,
                     l =>
                     _fetchBatchStrategy.Send(l, new FetchMessage
     {
         Topic     = topic,
         Partition = partition,
         Offset    = offset,
         MaxBytes  = _configuration.FetchMessageMaxBytes
     }));
 }
Beispiel #3
0
        /// <summary>
        /// Handle a produce message:
        ///  - check if expired,
        ///  - check if the topic is currenlty postponed, in which case we postpone the message,
        ///  - get the partitioner, using the default one if needed,
        ///  - if no partition is available for the topic we refresh metadata once,
        ///  - if still no partition are available we postpone the message else we route it.
        ///
        /// The MessageRouted event is raised in case of successful routing.
        /// </summary>
        /// <param name="produceMessage"></param>
        /// <returns></returns>
        private async Task HandleProduceMessage(ProduceMessage produceMessage)
        {
            if (produceMessage.ExpirationDate < DateTime.UtcNow)
            {
                OnMessageExpired(produceMessage);
                return;
            }

            var topic = produceMessage.Topic;

            if (IsPostponedTopic(topic))
            {
                PostponeMessage(produceMessage);
                return;
            }

            PartitionSelector selector;

            if (!_partitioners.TryGetValue(topic, out selector))
            {
                selector             = new PartitionSelector(_configuration.NumberOfMessagesBeforeRoundRobin, _randomGenerator.Next());
                _partitioners[topic] = selector;
            }

            var partitions = _routingTable.GetPartitions(topic);

            if (partitions.Length == 0)
            {
                await EnsureHasRoutingTable();

                partitions = _routingTable.GetPartitions(topic);
            }

            var partition = selector.GetPartition(produceMessage.RequiredPartition, partitions, GetFilter(topic));

            if (partition.Id == Partitions.None)
            {
                // Retry without filters because filtered partitions should be valid, we just wanted to avoid
                // spamming them while they're being rebalanced.
                partition = selector.GetPartition(produceMessage.RequiredPartition, partitions);

                if (partition.Id == Partitions.None)
                {
                    // So now there is really no available partition.
                    // Messages for topics with no partition available are postponed.
                    // They will be checked again when the routing table is updated.
                    PostponeMessage(produceMessage);
                    return;
                }
            }

            produceMessage.Partition = partition.Id;
            if (_batchStrategy.Send(partition.Leader, produceMessage))
            {
                MessageRouted(topic);
            }
            else
            {
                ReEnqueue(produceMessage);
            }
        }