Ejemplo n.º 1
0
        private void GetNextOffset()
        {
            long earliest, latest;

            _manager.RefreshAndGetOffset(
                KafkaConfig.VersionId,
                KafkaConfig.ClientId,
                KafkaConfig.NextCorrelationId(),
                _topicName,
                _partition,
                true,
                out earliest,
                out latest);

            switch (_nextOffset)
            {
            case (long)Offset.Earliest:
                _nextOffset = earliest;
                return;

            case (long)Offset.Latest:
                _nextOffset = latest;
                return;
            }

            _nextOffset = Math.Max(_nextOffset, earliest);
        }
Ejemplo n.º 2
0
 public TopicMetadata GetMetadata()
 {
     return(_zkConnect.CreateSimpleManager()
            .RefreshMetadata(
                KafkaConfig.VersionId,
                KafkaConfig.ClientId,
                KafkaConfig.NextCorrelationId(),
                _name,
                false));
 }
Ejemplo n.º 3
0
        public KafkaSimpleConsumerStream(IZookeeperConnection zkConnect, string topicName, int partition, long offset)
        {
            _topicName = topicName;
            _partition = partition;
            _manager   = zkConnect.CreateSimpleManager();

            _manager.RefreshMetadata(
                KafkaConfig.VersionId,
                KafkaConfig.ClientId,
                KafkaConfig.NextCorrelationId(),
                _topicName,
                true);

            _consumer = _manager.GetConsumer(topicName, partition);

            _thread = new Thread(RunConsumer);

            _nextOffset = offset;
        }
Ejemplo n.º 4
0
        private IEnumerable <MessageAndOffset> Fetch()
        {
            var       success    = false;
            var       retryCount = 0;
            const int maxRetry   = 3;

            while (!success && retryCount < maxRetry)
            {
                try
                {
                    var response = _consumer.Fetch(
                        KafkaConfig.ClientId,
                        _topicName,
                        KafkaConfig.NextCorrelationId(),
                        _partition,
                        _nextOffset,
                        ConsumerConfiguration.DefaultFetchSize,
                        ConsumerConfiguration.DefaultReceiveTimeout,
                        0);

                    if (response == null)
                    {
                        throw new KeyNotFoundException($"FetchRequest returned null response,fetchOffset={_nextOffset},leader={_consumer.Config.Broker},topic={_topicName},partition={_partition}");
                    }

                    var partitionData = response.PartitionData(_topicName, _partition);
                    if (partitionData == null)
                    {
                        throw new KeyNotFoundException($"PartitionData is null,fetchOffset={_nextOffset},leader={_consumer.Config.Broker},topic={_topicName},partition={_partition}");
                    }

                    if (partitionData.Error == ErrorMapping.OffsetOutOfRangeCode)
                    {
                        Logger.Warn($"PullMessage OffsetOutOfRangeCode,change to Latest,topic={_topicName},leader={_consumer.Config.Broker},partition={_partition},FetchOffset={_nextOffset},retryCount={retryCount},maxRetry={maxRetry}");
                        GetNextOffset();
                        return(null);
                    }

                    if (partitionData.Error != ErrorMapping.NoError)
                    {
                        Logger.Warn($"PullMessage ErrorCode={partitionData.Error},topic={_topicName},leader={_consumer.Config.Broker},partition={_partition},FetchOffset={_nextOffset},retryCount={retryCount},maxRetry={maxRetry}");
                        GetNextOffset();
                        return(null);
                    }

                    success = true;
                    var messages = partitionData.GetMessageAndOffsets();
                    if (messages == null || !messages.Any())
                    {
                        return(messages);
                    }

                    var count = messages.Count;

                    var lastOffset = messages.Last().MessageOffset;

                    if (count + _nextOffset != lastOffset + 1)
                    {
                        Logger.Warn($"PullMessage offset payloadCount out-of-sync,topic={_topicName},leader={_consumer.Config.Broker},partition={_partition},payloadCount={count},FetchOffset={_nextOffset},lastOffset={lastOffset},retryCount={retryCount},maxRetry={maxRetry}");
                        GetNextOffset();
                    }

                    return(messages);
                }
                catch (Exception)
                {
                    if (retryCount >= maxRetry)
                    {
                        throw;
                    }
                    GetNextOffset();
                }
                finally
                {
                    retryCount++;
                }
            }
            return(null);
        }