public void TestEmptyFetchRequest() { var partitionRequests = new Dictionary<TopicAndPartition, PartitionFetchInfo>(); var request = new FetchRequest(requestInfo: partitionRequests); var fetched = Consumer.Fetch(request); Assert.True(!fetched.HasError && fetched.Data.Count == 0); }
public FetchRequest Build() { var fetchRequest = new FetchRequest( this.versionId, this.correlationId.GetAndIncrement(), this.clientId, this.replicaId, this.maxWait, this.minBytes, new Dictionary<TopicAndPartition, PartitionFetchInfo>(this.requestMap)); this.requestMap.Clear(); return fetchRequest; }
protected bool Equals(FetchRequest other) { return this.VersionId == other.VersionId && string.Equals(this.ClientId, other.ClientId) && this.ReplicaId == other.ReplicaId && this.CorrelationId == other.CorrelationId && this.MaxWait == other.MaxWait && this.MinBytes == other.MinBytes && this.RequestInfo.DictionaryEqual(other.RequestInfo); }
/// <summary> /// Fetch a set of messages from a topic. /// </summary> /// <param name="request"> specifies the topic name, topic partition, starting byte offset, maximum bytes to be fetched.</param> /// <returns></returns> internal FetchResponse Fetch(FetchRequest request) { Receive response = null; var specificTimer = this.fetchRequestAndResponseStats.GetFetchRequestAndResponseStats(this.BrokerInfo).RequestTimer; var aggregateTimer = this.fetchRequestAndResponseStats.GetFetchRequestAndResponseAllBrokersStats().RequestTimer; aggregateTimer.Time(() => specificTimer.Time(() => { response = this.SendRequest(request); })); var fetchResponse = FetchResponse.ReadFrom(response.Buffer); var fetchedSize = fetchResponse.SizeInBytes; this.fetchRequestAndResponseStats.GetFetchRequestAndResponseStats(this.BrokerInfo).RequestSizeHist.Update(fetchedSize); this.fetchRequestAndResponseStats.GetFetchRequestAndResponseAllBrokersStats().RequestSizeHist.Update(fetchedSize); return fetchResponse; }
public void ProcessFetchRequest(FetchRequest fetchRequest) { var partitionsWithError = new HashSet<TopicAndPartition>(); FetchResponse response = null; try { Logger.DebugFormat("issuing to broker {0} of fetch request {1}", this.sourceBroker.Id, fetchRequest); response = this.simpleConsumer.Fetch(fetchRequest); } catch (Exception e) { if (isRunning.Get()) { Logger.Error("Error in fetch " + fetchRequest, e); this.partitionMapLock.Lock(); try { foreach (var key in this.partitionMap.Keys) { partitionsWithError.Add(key); } } finally { this.partitionMapLock.Unlock(); } } } this.FetcherStats.RequestRate.Mark(); if (response != null) { // process fetched Data this.partitionMapLock.Lock(); try { foreach (var topicAndData in response.Data) { var topicAndPartition = topicAndData.Key; var partitionData = topicAndData.Value; var topic = topicAndPartition.Topic; var partitionId = topicAndPartition.Partiton; long currentOffset; if (this.partitionMap.TryGetValue(topicAndPartition, out currentOffset) && fetchRequest.RequestInfo[topicAndPartition].Offset == currentOffset) { // we append to the log if the current offset is defined and it is the same as the offset requested during fetch switch (partitionData.Error) { case ErrorMapping.NoError: try { var messages = (ByteBufferMessageSet)partitionData.Messages; var messageAndOffset = messages.ShallowIterator().ToEnumerable().LastOrDefault(); var newOffset = messageAndOffset != null ? messageAndOffset.NextOffset : currentOffset; this.partitionMap[topicAndPartition] = newOffset; if (StatSettings.FetcherThreadStatsEnabled) { var validBytes = messages.ValidBytes; this.FetcherLagStats.GetFetcherLagStats(topic, partitionId).Lag = partitionData.Hw - newOffset; this.FetcherStats.ByteRate.Mark(validBytes); } // Once we hand off the partition Data to the subclass, we can't mess with it any more in this thread this.ProcessPartitionData(topicAndPartition, currentOffset, partitionData); } catch (InvalidMessageException ime) { // we log the error and continue. This ensures two things // 1. If there is a corrupt message in a topic partition, it does not bring the fetcher thread down and cause other topic partition to also lag // 2. If the message is corrupt due to a transient state in the log (truncation, partial writes can cause this), we simply continue and // should get fixed in the subsequent fetches Logger.ErrorFormat( "Found invalid messages during fetch for partiton [{0},{1}] offset {2} error {3}", topic, partitionId, currentOffset, ime.Message); } catch (Exception e) { throw new KafkaException( string.Format( "error processing Data for partition [{0},{1}] offset {2}", topic, partitionId, currentOffset), e); } break; case ErrorMapping.OffsetOutOfRangeCode: try { var newOffset = this.HandleOffsetOutOfRange(topicAndPartition); this.partitionMap[topicAndPartition] = newOffset; Logger.ErrorFormat( "Current offset {0} for partiton [{1},{2}] out of range; reste offset to {3}", currentOffset, topic, partitionId, newOffset); } catch (Exception e) { Logger.Error( string.Format( "Error getting offset for partiton [{0},{1}] to broker {2}", topic, partitionId, sourceBroker.Id), e); partitionsWithError.Add(topicAndPartition); } break; default: if (isRunning.Get()) { Logger.ErrorFormat( "Error for partition [{0},{1}] to broker {2}:{3}", topic, partitionId, this.sourceBroker.Id, ErrorMapping.ExceptionFor(partitionData.Error).GetType().Name); partitionsWithError.Add(topicAndPartition); } break; } } } } finally { this.partitionMapLock.Unlock(); } } if (partitionsWithError.Count > 0) { Logger.DebugFormat("handling partitions with error for {0}", string.Join(",", partitionsWithError)); this.HandlePartitionsWithErrors(partitionsWithError); } }
public RequestResponseSerializationTest() { this.producerRequest = SerializationTestUtils.CreateTestProducerRequest(); this.producerResponse = SerializationTestUtils.CreateTestProducerResponse(); this.fetchRequest = SerializationTestUtils.CreateTestFetchRequest(); this.offsetRequest = SerializationTestUtils.CreateTestOffsetRequest(); this.offsetResponse = SerializationTestUtils.CreateTestOffsetResponse(); this.topicMetadataRequest = SerializationTestUtils.CreateTestTopicMetadataRequest(); this.topicMetadataResponse = SerializationTestUtils.CreateTestTopicMetadataResponse(); }