/// <summary>
/// MANIFOLD use
/// </summary>
public FetchResponseWrapper FetchAndGetDetail(string clientId, string topic, int correlationId, int partitionId, long fetchOffset, int fetchSize
, int maxWaitTime, int minWaitSize)
{
FetchResponse response = this.Fetch(clientId,
topic,
correlationId,
partitionId,
fetchOffset,
fetchSize,
maxWaitTime,
minWaitSize);
if (response == null)
{
throw new KafkaConsumeException(string.Format("FetchRequest returned null FetchResponse,fetchOffset={0},leader={1},topic={2},partition={3}",
fetchOffset, this.Config.Broker, topic, partitionId));
}
PartitionData partitionData = response.PartitionData(topic, partitionId);
if (partitionData == null)
{
throw new KafkaConsumeException(string.Format("PartitionData int FetchResponse is null,fetchOffset={0},leader={1},topic={2},partition={3}",
fetchOffset, this.Config.Broker, topic, partitionId));
}
if (partitionData.Error != ErrorMapping.NoError)
{
string s = string.Format("Partition data in FetchResponse has error. {0} {1} fetchOffset={2},leader={3},topic={4},partition={5}"
, partitionData.Error, KafkaException.GetMessage(partitionData.Error)
, fetchOffset, this.Config.Broker, topic, partitionId);
Logger.Error(s);
throw new KafkaConsumeException(s, partitionData.Error);
}
return(new FetchResponseWrapper(partitionData.GetMessageAndOffsets(), response.Size, response.CorrelationId, topic, partitionId));
}
private static List <MessageAndOffset> FetchAndGetMessageAndOffsetList(
Consumer consumer,
int correlationID,
string topic,
int partitionIndex,
long fetchOffset,
int fetchSize,
int maxWaitTime,
int minWaitSize)
{
List <MessageAndOffset> listMessageAndOffsets = new List <MessageAndOffset>();
PartitionData partitionData = null;
int payloadCount = 0;
// at least retry once
int maxRetry = 1;
int retryCount = 0;
string s = string.Empty;
bool success = false;
while (!success && retryCount < maxRetry)
{
try
{
FetchResponse response = consumer.Fetch(Assembly.GetExecutingAssembly().ManifestModule.ToString(), // client id
topic,
correlationID, //random.Next(int.MinValue, int.MaxValue), // correlation id
partitionIndex,
fetchOffset,
fetchSize,
maxWaitTime,
minWaitSize);
if (response == null)
{
throw new KeyNotFoundException(string.Format("FetchRequest returned null response,fetchOffset={0},leader={1},topic={2},partition={3}",
fetchOffset, consumer.Config.Broker, topic, partitionIndex));
}
partitionData = response.PartitionData(topic, partitionIndex);
if (partitionData == null)
{
throw new KeyNotFoundException(string.Format("PartitionData is null,fetchOffset={0},leader={1},topic={2},partition={3}",
fetchOffset, consumer.Config.Broker, topic, partitionIndex));
}
if (partitionData.Error == ErrorMapping.OffsetOutOfRangeCode)
{
s = "PullMessage OffsetOutOfRangeCode,change to Latest,topic={0},leader={1},partition={2},FetchOffset={3},retryCount={4},maxRetry={5}";
Logger.ErrorFormat(s, topic, consumer.Config.Broker, partitionIndex, fetchOffset, retryCount, maxRetry);
return(null);
}
if (partitionData.Error != ErrorMapping.NoError)
{
s = "PullMessage ErrorCode={0},topic={1},leader={2},partition={3},FetchOffset={4},retryCount={5},maxRetry={6}";
Logger.ErrorFormat(s, partitionData.Error, topic, consumer.Config.Broker, partitionIndex, fetchOffset, retryCount, maxRetry);
return(null);
}
success = true;
listMessageAndOffsets = partitionData.GetMessageAndOffsets();
if (null != listMessageAndOffsets && listMessageAndOffsets.Any())
{
//TODO: When key are same for sequence of message, need count payloadCount by this way. So why line 438 work? is there bug?
payloadCount = listMessageAndOffsets.Count;// messages.Count();
long lastOffset = listMessageAndOffsets.Last().MessageOffset;
if ((payloadCount + fetchOffset) != (lastOffset + 1))
{
s = "PullMessage offset payloadCount out-of-sync,topic={0},leader={1},partition={2},payloadCount={3},FetchOffset={4},lastOffset={5},retryCount={6},maxRetry={7}";
Logger.ErrorFormat(s, topic, consumer.Config.Broker, partitionIndex, payloadCount, fetchOffset, lastOffset, retryCount, maxRetry);
}
}
return(listMessageAndOffsets);
}
catch (Exception)
{
if (retryCount >= maxRetry)
{
throw;
}
}
finally
{
retryCount++;
}
} // end of while loop
return(listMessageAndOffsets);
}
