private void MaybeCommitOffsets(IConsumer <string, string> consumer, KafkaMessageProcessor processor) { var logContext = $"{nameof(MaybeCommitOffsets)} for SubscriberId='{_subscriberId}'"; List <TopicPartitionOffset> offsetsToCommit = processor.OffsetsToCommit().ToList(); if (offsetsToCommit.Any()) { _logger.LogDebug($"{logContext}: Committing offsets='{String.Join(",", offsetsToCommit)}'"); consumer.Commit(offsetsToCommit); processor.NoteOffsetsCommitted(offsetsToCommit); _logger.LogDebug($"-{logContext}"); } }
public void Start() { var logContext = $"{nameof(Start)} for SubscriberId={_subscriberId}"; try { IConsumer <string, string> consumer = new ConsumerBuilder <string, string>(_consumerProperties).Build(); var processor = new KafkaMessageProcessor(_subscriberId, _handler, _loggerFactory.CreateLogger <KafkaMessageProcessor>()); using (IAdminClient adminClient = new DependentAdminClientBuilder(consumer.Handle).Build()) { foreach (string topic in _topics) { VerifyTopicExistsBeforeSubscribing(adminClient, topic); } } List <string> topicsList = new List <string>(_topics); _logger.LogDebug($"{logContext}: Subscribing to topics='{String.Join(",", topicsList)}'"); consumer.Subscribe(topicsList); // Set state to started before starting the processing thread instead of after as in the Java code // (prevent setting it to it started after it has potentially already been set to stopped) _state = EventuateKafkaConsumerState.Started; Task.Run(() => { try { while (!_cancellationTokenSource.IsCancellationRequested) { try { ConsumeResult <string, string> record = consumer.Consume(TimeSpan.FromMilliseconds(ConsumePollMilliseconds)); if (record != null) { _logger.LogDebug( $"{logContext}: process record at offset='{record.Offset}', key='{record.Key}', value='{record.Value}'"); processor.Process(record); } else { processor.ThrowExceptionIfHandlerFailed(); } MaybeCommitOffsets(consumer, processor); } catch (ConsumeException e) { _logger.LogError($"{logContext}: ConsumeException - {e.Error}. Continuing."); } } _state = EventuateKafkaConsumerState.Stopped; } catch (TaskCanceledException) { _logger.LogInformation($"{logContext}: Shutdown by cancel"); _state = EventuateKafkaConsumerState.Stopped; } catch (KafkaMessageProcessorFailedException e) { _logger.LogError($"{logContext}: Terminating due to KafkaMessageProcessorFailedException - {e}"); _state = EventuateKafkaConsumerState.MessageHandlingFailed; } catch (Exception e) { _logger.LogError($"{logContext}: Exception - {e}"); _state = EventuateKafkaConsumerState.Failed; // Java throws here, but seems like it isn't necessary } finally { // Try to put the last of the offsets away. Note that the // callbacks are done asynchronously so there is no guarantee // that all the offsets are ready. Worst case is that there // are messages processed more than once. MaybeCommitOffsets(consumer, processor); consumer.Dispose(); _logger.LogDebug($"{logContext}: Stopped in state {_state.ToString()}"); } }, _cancellationTokenSource.Token); } catch (Exception e) { _logger.LogError(e, $"{logContext}: Error subscribing"); _state = EventuateKafkaConsumerState.FailedToStart; throw; } }