public async Task <IList <IBatchContainer> > GetQueueMessagesAsync(int maxCount)
{
try
{
Task <IEnumerable <Message> > fetchingTask;
IList <IBatchContainer> batches = new List <IBatchContainer>();
using (TimerTimeToGetMessageFromKafka.NewContext(Id.ToString()))
{
fetchingTask = _consumer.FetchMessages(maxCount, CurrentOffset);
await Task.WhenAny(fetchingTask, Task.Delay(_options.ReceiveWaitTimeInMs));
}
// Checking that the task completed successfully
if (!fetchingTask.IsCompleted)
{
_logger.Warn((int)KafkaErrorCodes.KafkaStreamProviderBase,
"KafkaQueueAdapterReceiver - Fetching operation was not completed, tried to fetch {0} messages from offest {1}",
maxCount, CurrentOffset);
return(batches);
}
if (fetchingTask.IsFaulted && fetchingTask.Exception != null)
{
_logger.Warn((int)KafkaErrorCodes.KafkaStreamProviderBase,
"KafkaQueueAdapterReceiver - Fetching messages from kafka failed, tried to fetch {0} messages from offest {1}",
maxCount, CurrentOffset);
throw fetchingTask.Exception;
}
if (fetchingTask.Result == null)
{
return(batches);
}
var messages = fetchingTask.Result.ToList();
batches = messages.Select(m => _factory.FromKafkaMessage(m, m.Meta.Offset)).ToList();
// No batches, we are done here..
if (batches.Count <= 0)
{
return(batches);
}
_logger.Info("KafkaQueueAdapterReceiver - Pulled {0} messages for queue number {1}", batches.Count, Id.GetNumericId());
CurrentOffset += batches.Count;
// Taking a bit of metrics
MeterConsumedMessagesPerSecond.Mark(Id.ToString(), 1);
HistogramConsumedMessagesPerFetch.Update(batches.Count, Id.ToString());
return(batches);
}
catch (BufferUnderRunException)
{
// This case the next message in the queue is too big for us to read, so we skip it
_logger.Error((int)KafkaErrorCodes.KafkaStreamProviderBase, $"KafkaQueueAdapterReceiver - A message in the Kafka queue was too big to pull, skipping over it. offset was {CurrentOffset}");
CurrentOffset++;
return(new List <IBatchContainer>());
}
}
