public void TestEmptyFetchRequest()
{
var partitionRequests = new Dictionary<TopicAndPartition, PartitionFetchInfo>();
var request = new FetchRequest(requestInfo: partitionRequests);
var fetched = Consumer.Fetch(request);
Assert.True(!fetched.HasError && fetched.Data.Count == 0);
}
public FetchRequest Build()
{
var fetchRequest = new FetchRequest(
this.versionId,
this.correlationId.GetAndIncrement(),
this.clientId,
this.replicaId,
this.maxWait,
this.minBytes,
new Dictionary<TopicAndPartition, PartitionFetchInfo>(this.requestMap));
this.requestMap.Clear();
return fetchRequest;
}
protected bool Equals(FetchRequest other)
{
return this.VersionId == other.VersionId && string.Equals(this.ClientId, other.ClientId)
&& this.ReplicaId == other.ReplicaId && this.CorrelationId == other.CorrelationId
&& this.MaxWait == other.MaxWait && this.MinBytes == other.MinBytes
&& this.RequestInfo.DictionaryEqual(other.RequestInfo);
}
/// <summary>
/// Fetch a set of messages from a topic.
/// </summary>
/// <param name="request"> specifies the topic name, topic partition, starting byte offset, maximum bytes to be fetched.</param>
/// <returns></returns>
internal FetchResponse Fetch(FetchRequest request)
{
Receive response = null;
var specificTimer = this.fetchRequestAndResponseStats.GetFetchRequestAndResponseStats(this.BrokerInfo).RequestTimer;
var aggregateTimer = this.fetchRequestAndResponseStats.GetFetchRequestAndResponseAllBrokersStats().RequestTimer;
aggregateTimer.Time(() => specificTimer.Time(() => { response = this.SendRequest(request); }));
var fetchResponse = FetchResponse.ReadFrom(response.Buffer);
var fetchedSize = fetchResponse.SizeInBytes;
this.fetchRequestAndResponseStats.GetFetchRequestAndResponseStats(this.BrokerInfo).RequestSizeHist.Update(fetchedSize);
this.fetchRequestAndResponseStats.GetFetchRequestAndResponseAllBrokersStats().RequestSizeHist.Update(fetchedSize);
return fetchResponse;
}
public void ProcessFetchRequest(FetchRequest fetchRequest)
{
var partitionsWithError = new HashSet<TopicAndPartition>();
FetchResponse response = null;
try
{
Logger.DebugFormat("issuing to broker {0} of fetch request {1}", this.sourceBroker.Id, fetchRequest);
response = this.simpleConsumer.Fetch(fetchRequest);
}
catch (Exception e)
{
if (isRunning.Get())
{
Logger.Error("Error in fetch " + fetchRequest, e);
this.partitionMapLock.Lock();
try
{
foreach (var key in this.partitionMap.Keys)
{
partitionsWithError.Add(key);
}
}
finally
{
this.partitionMapLock.Unlock();
}
}
}
this.FetcherStats.RequestRate.Mark();
if (response != null)
{
// process fetched Data
this.partitionMapLock.Lock();
try
{
foreach (var topicAndData in response.Data)
{
var topicAndPartition = topicAndData.Key;
var partitionData = topicAndData.Value;
var topic = topicAndPartition.Topic;
var partitionId = topicAndPartition.Partiton;
long currentOffset;
if (this.partitionMap.TryGetValue(topicAndPartition, out currentOffset)
&& fetchRequest.RequestInfo[topicAndPartition].Offset == currentOffset)
{
// we append to the log if the current offset is defined and it is the same as the offset requested during fetch
switch (partitionData.Error)
{
case ErrorMapping.NoError:
try
{
var messages = (ByteBufferMessageSet)partitionData.Messages;
var messageAndOffset =
messages.ShallowIterator().ToEnumerable().LastOrDefault();
var newOffset = messageAndOffset != null
? messageAndOffset.NextOffset
: currentOffset;
this.partitionMap[topicAndPartition] = newOffset;
if (StatSettings.FetcherThreadStatsEnabled)
{
var validBytes = messages.ValidBytes;
this.FetcherLagStats.GetFetcherLagStats(topic, partitionId).Lag = partitionData.Hw
-
newOffset;
this.FetcherStats.ByteRate.Mark(validBytes);
}
// Once we hand off the partition Data to the subclass, we can't mess with it any more in this thread
this.ProcessPartitionData(topicAndPartition, currentOffset, partitionData);
}
catch (InvalidMessageException ime)
{
// we log the error and continue. This ensures two things
// 1. If there is a corrupt message in a topic partition, it does not bring the fetcher thread down and cause other topic partition to also lag
// 2. If the message is corrupt due to a transient state in the log (truncation, partial writes can cause this), we simply continue and
// should get fixed in the subsequent fetches
Logger.ErrorFormat(
"Found invalid messages during fetch for partiton [{0},{1}] offset {2} error {3}",
topic,
partitionId,
currentOffset,
ime.Message);
}
catch (Exception e)
{
throw new KafkaException(
string.Format(
"error processing Data for partition [{0},{1}] offset {2}",
topic,
partitionId,
currentOffset),
e);
}
break;
case ErrorMapping.OffsetOutOfRangeCode:
try
{
var newOffset = this.HandleOffsetOutOfRange(topicAndPartition);
this.partitionMap[topicAndPartition] = newOffset;
Logger.ErrorFormat(
"Current offset {0} for partiton [{1},{2}] out of range; reste offset to {3}",
currentOffset,
topic,
partitionId,
newOffset);
}
catch (Exception e)
{
Logger.Error(
string.Format(
"Error getting offset for partiton [{0},{1}] to broker {2}",
topic,
partitionId,
sourceBroker.Id),
e);
partitionsWithError.Add(topicAndPartition);
}
break;
default:
if (isRunning.Get())
{
Logger.ErrorFormat(
"Error for partition [{0},{1}] to broker {2}:{3}",
topic,
partitionId,
this.sourceBroker.Id,
ErrorMapping.ExceptionFor(partitionData.Error).GetType().Name);
partitionsWithError.Add(topicAndPartition);
}
break;
}
}
}
}
finally
{
this.partitionMapLock.Unlock();
}
}
if (partitionsWithError.Count > 0)
{
Logger.DebugFormat("handling partitions with error for {0}", string.Join(",", partitionsWithError));
this.HandlePartitionsWithErrors(partitionsWithError);
}
}
public RequestResponseSerializationTest()
{
this.producerRequest = SerializationTestUtils.CreateTestProducerRequest();
this.producerResponse = SerializationTestUtils.CreateTestProducerResponse();
this.fetchRequest = SerializationTestUtils.CreateTestFetchRequest();
this.offsetRequest = SerializationTestUtils.CreateTestOffsetRequest();
this.offsetResponse = SerializationTestUtils.CreateTestOffsetResponse();
this.topicMetadataRequest = SerializationTestUtils.CreateTestTopicMetadataRequest();
this.topicMetadataResponse = SerializationTestUtils.CreateTestTopicMetadataResponse();
}