void PumpTopic(string topic)
{
var cancellationToken = _cancellationTokenSource.Token;
_logger.Info("Starting consumer worker for topic {topic}", topic);
try
{
var topicDirectoryPath = Path.Combine(_directoryPath, topic);
var logDirectory = new LogDirectory(topicDirectoryPath, new Settings(logger: new KafkaesqueToToposLogger(_logger)));
var reader = logDirectory.GetReader();
while (!cancellationToken.IsCancellationRequested)
{
try
{
var resumePosition = _positionManager.Get(topic, 0).Result;
var(fileNumber, bytePosition) = resumePosition.ToKafkaesquePosition();
_logger.Debug("Resuming consumer from file {fileNumber} byte {bytePosition}", fileNumber, bytePosition);
foreach (var eventData in reader.Read(fileNumber, bytePosition, cancellationToken: cancellationToken))
{
var transportMessage = JsonConvert.DeserializeObject <TransportMessage>(Encoding.UTF8.GetString(eventData.Data));
var kafkaesqueEventPosition = new KafkaesquePosition(eventData.FileNumber, eventData.BytePosition);
var eventPosition = kafkaesqueEventPosition.ToPosition(topic, partition: 0);
var receivedTransportMessage = new ReceivedTransportMessage(eventPosition, transportMessage.Headers, transportMessage.Body);
_logger.Debug("Received event {position}", eventPosition);
_consumerDispatcher.Dispatch(receivedTransportMessage);
}
}
catch (Exception exception)
{
_logger.Warn(exception, "Error in consumer worker for topic {topic} - waiting 10 s", topic);
Task.Delay(TimeSpan.FromSeconds(10), cancellationToken)
.Wait(cancellationToken);
}
}
}
catch (OperationCanceledException) when(cancellationToken.IsCancellationRequested)
{
// we're done
}
catch (Exception exception)
{
_logger.Error(exception, "Unhandled exception in consumer worker for topic {topic}", topic);
}
finally
{
_logger.Info("Stopped consumer worker for topic {topic}", topic);
}
}
void TryProcessNextMessage(IConsumer <string, byte[]> consumer, CancellationToken cancellationToken)
{
try
{
var consumeResult = consumer.Consume(cancellationToken);
if (consumeResult == null)
{
Thread.Sleep(100); //< chill (but it should not happen)
return;
}
var position = new Position(
topic: consumeResult.Topic,
partition: consumeResult.Partition.Value,
offset: consumeResult.Offset.Value
);
var message = new ReceivedTransportMessage(
position: position,
headers: GetHeaders(consumeResult.Headers),
body: consumeResult.Value
);
_logger.Debug("Received event {position}", position);
_consumerDispatcher.Dispatch(message);
}
catch (OperationCanceledException) when(cancellationToken.IsCancellationRequested)
{
// it's alright
}
catch (ThreadAbortException)
{
_logger.Warn("Kafka consumer worker aborted!");
}
catch (Exception exception)
{
_logger.Warn(exception, "Unhandled exception in Kafka consumer loop - waiting 30 s");
try
{
Task.Delay(TimeSpan.FromSeconds(30), cancellationToken)
.Wait(cancellationToken);
}
catch (OperationCanceledException) when(cancellationToken.IsCancellationRequested)
{
}
}
}
