private void MaybeCommitOffsets(IConsumer <string, string> consumer, KafkaMessageProcessor processor)
{
var logContext = $"{nameof(MaybeCommitOffsets)} for SubscriberId='{_subscriberId}'";
List <TopicPartitionOffset> offsetsToCommit = processor.OffsetsToCommit().ToList();
if (offsetsToCommit.Any())
{
_logger.LogDebug($"{logContext}: Committing offsets='{String.Join(",", offsetsToCommit)}'");
consumer.Commit(offsetsToCommit);
processor.NoteOffsetsCommitted(offsetsToCommit);
_logger.LogDebug($"-{logContext}");
}
}
public void Start()
{
var logContext = $"{nameof(Start)} for SubscriberId={_subscriberId}";
try
{
IConsumer <string, string> consumer = new ConsumerBuilder <string, string>(_consumerProperties).Build();
var processor = new KafkaMessageProcessor(_subscriberId, _handler,
_loggerFactory.CreateLogger <KafkaMessageProcessor>());
using (IAdminClient adminClient = new DependentAdminClientBuilder(consumer.Handle).Build())
{
foreach (string topic in _topics)
{
VerifyTopicExistsBeforeSubscribing(adminClient, topic);
}
}
List <string> topicsList = new List <string>(_topics);
_logger.LogDebug($"{logContext}: Subscribing to topics='{String.Join(",", topicsList)}'");
consumer.Subscribe(topicsList);
// Set state to started before starting the processing thread instead of after as in the Java code
// (prevent setting it to it started after it has potentially already been set to stopped)
_state = EventuateKafkaConsumerState.Started;
Task.Run(() =>
{
try
{
while (!_cancellationTokenSource.IsCancellationRequested)
{
try
{
ConsumeResult <string, string> record =
consumer.Consume(TimeSpan.FromMilliseconds(ConsumePollMilliseconds));
if (record != null)
{
_logger.LogDebug(
$"{logContext}: process record at offset='{record.Offset}', key='{record.Key}', value='{record.Value}'");
processor.Process(record);
}
else
{
processor.ThrowExceptionIfHandlerFailed();
}
MaybeCommitOffsets(consumer, processor);
}
catch (ConsumeException e)
{
_logger.LogError($"{logContext}: ConsumeException - {e.Error}. Continuing.");
}
}
_state = EventuateKafkaConsumerState.Stopped;
}
catch (TaskCanceledException)
{
_logger.LogInformation($"{logContext}: Shutdown by cancel");
_state = EventuateKafkaConsumerState.Stopped;
}
catch (KafkaMessageProcessorFailedException e)
{
_logger.LogError($"{logContext}: Terminating due to KafkaMessageProcessorFailedException - {e}");
_state = EventuateKafkaConsumerState.MessageHandlingFailed;
}
catch (Exception e)
{
_logger.LogError($"{logContext}: Exception - {e}");
_state = EventuateKafkaConsumerState.Failed;
// Java throws here, but seems like it isn't necessary
}
finally
{
// Try to put the last of the offsets away. Note that the
// callbacks are done asynchronously so there is no guarantee
// that all the offsets are ready. Worst case is that there
// are messages processed more than once.
MaybeCommitOffsets(consumer, processor);
consumer.Dispose();
_logger.LogDebug($"{logContext}: Stopped in state {_state.ToString()}");
}
}, _cancellationTokenSource.Token);
}
catch (Exception e)
{
_logger.LogError(e, $"{logContext}: Error subscribing");
_state = EventuateKafkaConsumerState.FailedToStart;
throw;
}
}