Beispiel #1
0
 public void TestEmptyFetchRequest()
 {
     var partitionRequests = new Dictionary<TopicAndPartition, PartitionFetchInfo>();
     var request = new FetchRequest(requestInfo: partitionRequests);
     var fetched = Consumer.Fetch(request);
     Assert.True(!fetched.HasError && fetched.Data.Count == 0);
 }
Beispiel #2
0
        public FetchRequest Build()
        {
            var fetchRequest = new FetchRequest(
                this.versionId,
                this.correlationId.GetAndIncrement(),
                this.clientId,
                this.replicaId,
                this.maxWait,
                this.minBytes,
                new Dictionary<TopicAndPartition, PartitionFetchInfo>(this.requestMap));

            this.requestMap.Clear();
            return fetchRequest;
        }
Beispiel #3
0
 protected bool Equals(FetchRequest other)
 {
     return this.VersionId == other.VersionId && string.Equals(this.ClientId, other.ClientId)
            && this.ReplicaId == other.ReplicaId && this.CorrelationId == other.CorrelationId
            && this.MaxWait == other.MaxWait && this.MinBytes == other.MinBytes
            && this.RequestInfo.DictionaryEqual(other.RequestInfo);
 }
Beispiel #4
0
        /// <summary>
        /// Fetch a set of messages from a topic.
        /// </summary>
        /// <param name="request"> specifies the topic name, topic partition, starting byte offset, maximum bytes to be fetched.</param>
        /// <returns></returns>
        internal FetchResponse Fetch(FetchRequest request)
        {
            Receive response = null;
            var specificTimer = this.fetchRequestAndResponseStats.GetFetchRequestAndResponseStats(this.BrokerInfo).RequestTimer;
            var aggregateTimer = this.fetchRequestAndResponseStats.GetFetchRequestAndResponseAllBrokersStats().RequestTimer;
            aggregateTimer.Time(() => specificTimer.Time(() => { response = this.SendRequest(request); }));

            var fetchResponse = FetchResponse.ReadFrom(response.Buffer);
            var fetchedSize = fetchResponse.SizeInBytes;

            this.fetchRequestAndResponseStats.GetFetchRequestAndResponseStats(this.BrokerInfo).RequestSizeHist.Update(fetchedSize);
            this.fetchRequestAndResponseStats.GetFetchRequestAndResponseAllBrokersStats().RequestSizeHist.Update(fetchedSize);

            return fetchResponse;
        }
        public void ProcessFetchRequest(FetchRequest fetchRequest)
        {
            var partitionsWithError = new HashSet<TopicAndPartition>();
            FetchResponse response = null;
            try
            {
                Logger.DebugFormat("issuing to broker {0} of fetch request {1}", this.sourceBroker.Id, fetchRequest);
                response = this.simpleConsumer.Fetch(fetchRequest);
            }
            catch (Exception e)
            {
                if (isRunning.Get())
                {
                    Logger.Error("Error in fetch " + fetchRequest, e);
                    this.partitionMapLock.Lock();
                    try
                    {
                        foreach (var key in this.partitionMap.Keys)
                        {
                            partitionsWithError.Add(key);
                        }
                    }
                    finally
                    {
                        this.partitionMapLock.Unlock();
                    }
                }
            }

            this.FetcherStats.RequestRate.Mark();

            if (response != null)
            {
                // process fetched Data
                this.partitionMapLock.Lock();
                try
                {
                    foreach (var topicAndData in response.Data)
                    {
                        var topicAndPartition = topicAndData.Key;
                        var partitionData = topicAndData.Value;
                        var topic = topicAndPartition.Topic;
                        var partitionId = topicAndPartition.Partiton;
                        long currentOffset;
                        if (this.partitionMap.TryGetValue(topicAndPartition, out currentOffset)
                            && fetchRequest.RequestInfo[topicAndPartition].Offset == currentOffset)
                        {
                            // we append to the log if the current offset is defined and it is the same as the offset requested during fetch
                            switch (partitionData.Error)
                            {
                                case ErrorMapping.NoError:
                                    try
                                    {
                                        var messages = (ByteBufferMessageSet)partitionData.Messages;
                                        var messageAndOffset =
                                            messages.ShallowIterator().ToEnumerable().LastOrDefault();
                                        var newOffset = messageAndOffset != null
                                                            ? messageAndOffset.NextOffset
                                                            : currentOffset;

                                        this.partitionMap[topicAndPartition] = newOffset;

                                        if (StatSettings.FetcherThreadStatsEnabled)
                                        {
                                            var validBytes = messages.ValidBytes;

                                            this.FetcherLagStats.GetFetcherLagStats(topic, partitionId).Lag = partitionData.Hw
                                                                                                              -
                                                                                                              newOffset;
                                            this.FetcherStats.ByteRate.Mark(validBytes);
                                        }

                                        // Once we hand off the partition Data to the subclass, we can't mess with it any more in this thread
                                        this.ProcessPartitionData(topicAndPartition, currentOffset, partitionData);
                                    }
                                    catch (InvalidMessageException ime)
                                    {
                                        // we log the error and continue. This ensures two things
                                        // 1. If there is a corrupt message in a topic partition, it does not bring the fetcher thread down and cause other topic partition to also lag
                                        // 2. If the message is corrupt due to a transient state in the log (truncation, partial writes can cause this), we simply continue and
                                        //    should get fixed in the subsequent fetches
                                        Logger.ErrorFormat(
                                            "Found invalid messages during fetch for partiton [{0},{1}] offset {2} error {3}",
                                            topic,
                                            partitionId,
                                            currentOffset,
                                            ime.Message);
                                    }
                                    catch (Exception e)
                                    {
                                        throw new KafkaException(
                                            string.Format(
                                                "error processing Data for partition [{0},{1}] offset {2}",
                                                topic,
                                                partitionId,
                                                currentOffset),
                                            e);
                                    }

                                    break;
                                case ErrorMapping.OffsetOutOfRangeCode:
                                    try
                                    {
                                        var newOffset = this.HandleOffsetOutOfRange(topicAndPartition);
                                        this.partitionMap[topicAndPartition] = newOffset;
                                        Logger.ErrorFormat(
                                            "Current offset {0} for partiton [{1},{2}] out of range; reste offset to {3}",
                                            currentOffset,
                                            topic,
                                            partitionId,
                                            newOffset);
                                    }
                                    catch (Exception e)
                                    {
                                        Logger.Error(
                                            string.Format(
                                                "Error getting offset for partiton [{0},{1}] to broker {2}",
                                                topic,
                                                partitionId,
                                                sourceBroker.Id),
                                            e);
                                        partitionsWithError.Add(topicAndPartition);
                                    }

                                    break;
                                default:
                                    if (isRunning.Get())
                                    {
                                        Logger.ErrorFormat(
                                            "Error for partition [{0},{1}] to broker {2}:{3}",
                                            topic,
                                            partitionId,
                                            this.sourceBroker.Id,
                                            ErrorMapping.ExceptionFor(partitionData.Error).GetType().Name);
                                        partitionsWithError.Add(topicAndPartition);
                                    }

                                    break;
                            }
                        }
                    }
                }
                finally
                {
                    this.partitionMapLock.Unlock();
                }
            }

            if (partitionsWithError.Count > 0)
            {
                Logger.DebugFormat("handling partitions with error for {0}", string.Join(",", partitionsWithError));
                this.HandlePartitionsWithErrors(partitionsWithError);
            }
        }
 public RequestResponseSerializationTest()
 {
     this.producerRequest = SerializationTestUtils.CreateTestProducerRequest();
     this.producerResponse = SerializationTestUtils.CreateTestProducerResponse();
     this.fetchRequest = SerializationTestUtils.CreateTestFetchRequest();
     this.offsetRequest = SerializationTestUtils.CreateTestOffsetRequest();
     this.offsetResponse = SerializationTestUtils.CreateTestOffsetResponse();
     this.topicMetadataRequest = SerializationTestUtils.CreateTestTopicMetadataRequest();
     this.topicMetadataResponse = SerializationTestUtils.CreateTestTopicMetadataResponse();
 }