private Task ConsumeTopicPartitionAsync(string topic, int partitionId)
{
return Task.Run(async () =>
{
try
{
bool refreshMetaData = false;
var bufferSizeHighWatermark = FetchRequest.DefaultBufferSize;
_options.Log.DebugFormat("Consumer: Creating polling task for topic: {0} on parition: {1}", topic, partitionId);
while (_disposeToken.IsCancellationRequested == false)
{
if (refreshMetaData)
{
await _options.Router.RefreshTopicMetadata(topic);
EnsurePartitionPollingThreads();
refreshMetaData = false;
}
try
{
//get the current offset, or default to zero if not there.
long offset = 0;
_partitionOffsetIndex.AddOrUpdate(partitionId, i => offset, (i, currentOffset) =>
{
offset = currentOffset;
return currentOffset;
});
//build a fetch request for partition at offset
var fetch = new Fetch
{
Topic = topic,
PartitionId = partitionId,
Offset = offset,
MaxBytes = bufferSizeHighWatermark,
};
var fetchRequest = fetch.ToFetchRequest(_options);
//make request and post to queue
var route = _options.Router.SelectBrokerRouteFromLocalCache(topic, partitionId);
var taskSend = route.Connection.SendAsync(fetchRequest);
await Task.WhenAny(taskSend, _disposeTask.Task).ConfigureAwait(false);
if (_disposeTask.Task.IsCompleted) return;
var responses = await taskSend; //already done
if (responses.Count > 0)
{
var response = responses.FirstOrDefault(); //we only asked for one response
var hasProcessed = await ProcessResponseAndUpdateOffsetAsync(response, fetch, partitionId);
if (_disposeToken.IsCancellationRequested) return;
// sleep is not needed if responses were received
if (hasProcessed) continue;
}
//no message received from server wait a while before we try another long poll
await Task.Delay(_options.BackoffInterval, _disposeToken.Token);
}
catch (BufferUnderRunException ex)
{
bufferSizeHighWatermark = (int)(ex.RequiredBufferSize * _options.FetchBufferMultiplier) +
ex.MessageHeaderSize;
_options.Log.InfoFormat("Buffer underrun. Increasing buffer size to: {0}",
bufferSizeHighWatermark);
}
catch (OffsetOutOfRangeException ex)
{
//TODO this turned out really ugly. Need to fix this section.
_options.Log.ErrorFormat(ex.Message);
FixOffsetOutOfRangeExceptionAsync(ex.FetchRequest);
}
catch (InvalidMetadataException ex)
{
//refresh our metadata and ensure we are polling the correct partitions
refreshMetaData = true;
_options.Log.ErrorFormat(ex.Message);
}
catch (TaskCanceledException ex)
{
//TODO :LOG
}
catch (Exception ex)
{
_options.Log.ErrorFormat("Exception occured while polling topic:{0} partition:{1}. Polling will continue. Exception={2}", topic, partitionId, ex);
}
}
}
finally
{
_options.Log.DebugFormat("Consumer: Disabling polling task for topic: {0} on parition: {1}", topic, partitionId);
Task tempTask;
_partitionPollingIndex.TryRemove(partitionId, out tempTask);
}
});
}