private async Task FixOffsetOutOfRangeExceptionAsync(FetchRequest.Topic topic)
{
await _options.Router.GetTopicOffsetsAsync(topic.TopicName, 2, -1, CancellationToken.None)
.ContinueWith(t =>
{
try
{
var offsets = t.Result.Where(x => x.PartitionId == topic.PartitionId).ToList();
if (!offsets.Any())
{
return;
}
var minOffset = offsets.Min(o => o.Offset);
if (minOffset > topic.Offset)
{
SetOffsetPosition(new OffsetPosition(topic.PartitionId, minOffset));
}
var maxOffset = offsets.Max(o => o.Offset);
if (maxOffset < topic.Offset)
{
SetOffsetPosition(new OffsetPosition(topic.PartitionId, maxOffset));
}
}
catch (Exception ex)
{
_options.Log.Error(LogEvent.Create(ex, $"Failed to fix the offset out of range exception on topic/{topic.TopicName}/partition/{topic.PartitionId}: Polling will continue"));
}
});
}
public async Task CanFetch()
{
int partitionId = 0;
var router = new BrokerRouter(new KafkaOptions(TestConfig.IntegrationUri));
var producer = new Producer(router);
string messageValue = Guid.NewGuid().ToString();
var response = await producer.SendMessageAsync(new Message(messageValue), TestConfig.TopicName(), partitionId, CancellationToken.None);
var offset = response.Offset;
var fetch = new FetchRequest.Topic(TestConfig.TopicName(), partitionId, offset, 32000);
var fetchRequest = new FetchRequest(fetch, minBytes: 10);
var r = await router.SendAsync(fetchRequest, TestConfig.TopicName(), partitionId, CancellationToken.None);
Assert.IsTrue(r.Topics.First().Messages.First().Value.ToUtf8String() == messageValue);
}
public async Task CanFetch()
{
const int partitionId = 0;
using (var router = await TestConfig.IntegrationOptions.CreateRouterAsync()) {
await router.TemporaryTopicAsync(async topicName => {
using (var producer = new Producer(router)) {
var messageValue = Guid.NewGuid().ToString();
var response = await producer.SendAsync(new Message(messageValue), TestConfig.TopicName(), partitionId, CancellationToken.None);
var offset = response.base_offset;
var fetch = new FetchRequest.Topic(TestConfig.TopicName(), partitionId, offset, 32000);
var fetchRequest = new FetchRequest(fetch, minBytes: 10);
var r = await router.SendAsync(fetchRequest, TestConfig.TopicName(), partitionId, CancellationToken.None);
Assert.IsTrue(r.responses.First().Messages.First().Value.ToUtf8String() == messageValue);
}
});
}
}
internal static async Task <ImmutableList <Message> > FetchMessagesAsync(this IRouter router, string topicName, int partitionId, long offset, IConsumerConfiguration configuration, CancellationToken cancellationToken)
{
var topic = new FetchRequest.Topic(topicName, partitionId, offset, configuration.MaxPartitionFetchBytes);
FetchResponse response = null;
for (var attempt = 1; response == null && attempt <= 12; attempt++) // at a (minimum) multiplier of 2, this results in a total factor of 256
{
var request = new FetchRequest(topic, configuration.MaxFetchServerWait, configuration.MinFetchBytes, configuration.MaxFetchBytes);
try {
response = await router.SendAsync(request, topicName, partitionId, cancellationToken).ConfigureAwait(false);
} catch (BufferUnderRunException ex) {
if (configuration.FetchByteMultiplier <= 1)
{
throw;
}
var maxBytes = topic.max_bytes * configuration.FetchByteMultiplier;
router.Log.Warn(() => LogEvent.Create(ex, $"Retrying Fetch Request with multiplier {Math.Pow(configuration.FetchByteMultiplier, attempt)}, {topic.max_bytes} -> {maxBytes}"));
topic = new FetchRequest.Topic(topic.topic, topic.partition_id, topic.fetch_offset, maxBytes);
}
}
return(response?.responses?.SingleOrDefault()?.Messages?.ToImmutableList() ?? ImmutableList <Message> .Empty);
}
private Task ConsumeTopicPartitionAsync(string topic, int partitionId, CancellationToken cancellationToken)
{
bool needToRefreshMetadata = false;
return(Task.Run(async() =>
{
try
{
var bufferSizeHighWatermark = FetchRequest.DefaultBufferSize;
_options.Log.Info(() => LogEvent.Create($"Consumer: Creating polling task for topic/{topic}/parition/{partitionId}"));
while (_disposeToken.IsCancellationRequested == false)
{
try
{
//after error
if (needToRefreshMetadata)
{
await _options.Router.GetTopicMetadataAsync(topic, cancellationToken).ConfigureAwait(false);
EnsurePartitionPollingThreads();
needToRefreshMetadata = false;
}
//get the current offset, or default to zero if not there.
long offset = 0;
_partitionOffsetIndex.AddOrUpdate(partitionId, i => offset, (i, currentOffset) =>
{
offset = currentOffset;
return currentOffset;
});
//build a fetch request for partition at offset
var fetch = new FetchRequest.Topic(topic, partitionId, offset, bufferSizeHighWatermark);
var fetchRequest = new FetchRequest(fetch, _options.MaxWaitTimeForMinimumBytes, _options.MinimumBytes);
//make request and post to queue
var route = _options.Router.GetBrokerRoute(topic, partitionId);
var taskSend = route.Connection.SendAsync(fetchRequest, cancellationToken);
await Task.WhenAny(taskSend, _disposeTask.Task).ConfigureAwait(false);
if (_disposeTask.Task.IsCompleted)
{
return;
}
//already done
var response = await taskSend;
if (response?.Topics.Count > 0)
{
// we only asked for one response
var fetchTopicResponse = response.Topics.FirstOrDefault();
if (fetchTopicResponse != null && fetchTopicResponse.Messages.Any())
{
HandleResponseErrors(fetchRequest, fetchTopicResponse, route.Connection);
foreach (var message in fetchTopicResponse.Messages)
{
await Task.Run(() => {
// this is a block operation
_fetchResponseQueue.Add(message, _disposeToken.Token);
}, _disposeToken.Token).ConfigureAwait(false);
if (_disposeToken.IsCancellationRequested)
{
return;
}
}
var nextOffset = fetchTopicResponse.Messages.Last().Offset + 1;
_partitionOffsetIndex.AddOrUpdate(partitionId, i => nextOffset, (i, l) => nextOffset);
// sleep is not needed if responses were received
continue;
}
}
//no message received from server wait a while before we try another long poll
await Task.Delay(_options.BackoffInterval, _disposeToken.Token);
}
catch (ConnectionException ex)
{
needToRefreshMetadata = true;
_options.Log.Error(LogEvent.Create(ex));
}
catch (BufferUnderRunException ex)
{
bufferSizeHighWatermark = (int)(ex.RequiredBufferSize * _options.FetchBufferMultiplier) + ex.MessageHeaderSize;
// ReSharper disable once AccessToModifiedClosure
_options.Log.Info(() => LogEvent.Create($"Buffer underrun: Increasing buffer size to {bufferSizeHighWatermark}"));
}
catch (FetchOutOfRangeException ex) when(ex.ErrorCode == ErrorResponseCode.OffsetOutOfRange)
{
//TODO this turned out really ugly. Need to fix this section.
_options.Log.Error(LogEvent.Create(ex));
await FixOffsetOutOfRangeExceptionAsync(ex.Topic);
}
catch (CachedMetadataException ex)
{
// refresh our metadata and ensure we are polling the correct partitions
needToRefreshMetadata = true;
_options.Log.Error(LogEvent.Create(ex));
}
catch (TaskCanceledException)
{
//TODO :LOG
}
catch (Exception ex)
{
_options.Log.Error(LogEvent.Create(ex, $"Failed polling topic/{topic}/partition/{partitionId}: Polling will continue"));
}
}
}
finally
{
_options.Log.Info(() => LogEvent.Create($"Consumer disabling polling task for topic/{topic}/partition/{partitionId}"));
Task tempTask;
_partitionPollingIndex.TryRemove(partitionId, out tempTask);
}
}, cancellationToken));
}