void PumpTopic(string topic) { var cancellationToken = _cancellationTokenSource.Token; _logger.Info("Starting consumer worker for topic {topic}", topic); try { var topicDirectoryPath = Path.Combine(_directoryPath, topic); var logDirectory = new LogDirectory(topicDirectoryPath, new Settings(logger: new KafkaesqueToToposLogger(_logger))); var reader = logDirectory.GetReader(); while (!cancellationToken.IsCancellationRequested) { try { var resumePosition = _positionManager.Get(topic, 0).Result; var(fileNumber, bytePosition) = resumePosition.ToKafkaesquePosition(); _logger.Debug("Resuming consumer from file {fileNumber} byte {bytePosition}", fileNumber, bytePosition); foreach (var eventData in reader.Read(fileNumber, bytePosition, cancellationToken: cancellationToken)) { var transportMessage = JsonConvert.DeserializeObject <TransportMessage>(Encoding.UTF8.GetString(eventData.Data)); var kafkaesqueEventPosition = new KafkaesquePosition(eventData.FileNumber, eventData.BytePosition); var eventPosition = kafkaesqueEventPosition.ToPosition(topic, partition: 0); var receivedTransportMessage = new ReceivedTransportMessage(eventPosition, transportMessage.Headers, transportMessage.Body); _logger.Debug("Received event {position}", eventPosition); _consumerDispatcher.Dispatch(receivedTransportMessage); } } catch (Exception exception) { _logger.Warn(exception, "Error in consumer worker for topic {topic} - waiting 10 s", topic); Task.Delay(TimeSpan.FromSeconds(10), cancellationToken) .Wait(cancellationToken); } } } catch (OperationCanceledException) when(cancellationToken.IsCancellationRequested) { // we're done } catch (Exception exception) { _logger.Error(exception, "Unhandled exception in consumer worker for topic {topic}", topic); } finally { _logger.Info("Stopped consumer worker for topic {topic}", topic); } }
void TryProcessNextMessage(IConsumer <string, byte[]> consumer, CancellationToken cancellationToken) { try { var consumeResult = consumer.Consume(cancellationToken); if (consumeResult == null) { Thread.Sleep(100); //< chill (but it should not happen) return; } var position = new Position( topic: consumeResult.Topic, partition: consumeResult.Partition.Value, offset: consumeResult.Offset.Value ); var message = new ReceivedTransportMessage( position: position, headers: GetHeaders(consumeResult.Headers), body: consumeResult.Value ); _logger.Debug("Received event {position}", position); _consumerDispatcher.Dispatch(message); } catch (OperationCanceledException) when(cancellationToken.IsCancellationRequested) { // it's alright } catch (ThreadAbortException) { _logger.Warn("Kafka consumer worker aborted!"); } catch (Exception exception) { _logger.Warn(exception, "Unhandled exception in Kafka consumer loop - waiting 30 s"); try { Task.Delay(TimeSpan.FromSeconds(30), cancellationToken) .Wait(cancellationToken); } catch (OperationCanceledException) when(cancellationToken.IsCancellationRequested) { } } }