private void GetNextOffset()
{
long earliest, latest;
_manager.RefreshAndGetOffset(
KafkaConfig.VersionId,
KafkaConfig.ClientId,
KafkaConfig.NextCorrelationId(),
_topicName,
_partition,
true,
out earliest,
out latest);
switch (_nextOffset)
{
case (long)Offset.Earliest:
_nextOffset = earliest;
return;
case (long)Offset.Latest:
_nextOffset = latest;
return;
}
_nextOffset = Math.Max(_nextOffset, earliest);
}
public TopicMetadata GetMetadata()
{
return(_zkConnect.CreateSimpleManager()
.RefreshMetadata(
KafkaConfig.VersionId,
KafkaConfig.ClientId,
KafkaConfig.NextCorrelationId(),
_name,
false));
}
public KafkaSimpleConsumerStream(IZookeeperConnection zkConnect, string topicName, int partition, long offset)
{
_topicName = topicName;
_partition = partition;
_manager = zkConnect.CreateSimpleManager();
_manager.RefreshMetadata(
KafkaConfig.VersionId,
KafkaConfig.ClientId,
KafkaConfig.NextCorrelationId(),
_topicName,
true);
_consumer = _manager.GetConsumer(topicName, partition);
_thread = new Thread(RunConsumer);
_nextOffset = offset;
}
private IEnumerable <MessageAndOffset> Fetch()
{
var success = false;
var retryCount = 0;
const int maxRetry = 3;
while (!success && retryCount < maxRetry)
{
try
{
var response = _consumer.Fetch(
KafkaConfig.ClientId,
_topicName,
KafkaConfig.NextCorrelationId(),
_partition,
_nextOffset,
ConsumerConfiguration.DefaultFetchSize,
ConsumerConfiguration.DefaultReceiveTimeout,
0);
if (response == null)
{
throw new KeyNotFoundException($"FetchRequest returned null response,fetchOffset={_nextOffset},leader={_consumer.Config.Broker},topic={_topicName},partition={_partition}");
}
var partitionData = response.PartitionData(_topicName, _partition);
if (partitionData == null)
{
throw new KeyNotFoundException($"PartitionData is null,fetchOffset={_nextOffset},leader={_consumer.Config.Broker},topic={_topicName},partition={_partition}");
}
if (partitionData.Error == ErrorMapping.OffsetOutOfRangeCode)
{
Logger.Warn($"PullMessage OffsetOutOfRangeCode,change to Latest,topic={_topicName},leader={_consumer.Config.Broker},partition={_partition},FetchOffset={_nextOffset},retryCount={retryCount},maxRetry={maxRetry}");
GetNextOffset();
return(null);
}
if (partitionData.Error != ErrorMapping.NoError)
{
Logger.Warn($"PullMessage ErrorCode={partitionData.Error},topic={_topicName},leader={_consumer.Config.Broker},partition={_partition},FetchOffset={_nextOffset},retryCount={retryCount},maxRetry={maxRetry}");
GetNextOffset();
return(null);
}
success = true;
var messages = partitionData.GetMessageAndOffsets();
if (messages == null || !messages.Any())
{
return(messages);
}
var count = messages.Count;
var lastOffset = messages.Last().MessageOffset;
if (count + _nextOffset != lastOffset + 1)
{
Logger.Warn($"PullMessage offset payloadCount out-of-sync,topic={_topicName},leader={_consumer.Config.Broker},partition={_partition},payloadCount={count},FetchOffset={_nextOffset},lastOffset={lastOffset},retryCount={retryCount},maxRetry={maxRetry}");
GetNextOffset();
}
return(messages);
}
catch (Exception)
{
if (retryCount >= maxRetry)
{
throw;
}
GetNextOffset();
}
finally
{
retryCount++;
}
}
return(null);
}