void PumpTopic(string topic)
{
var cancellationToken = _cancellationTokenSource.Token;
_logger.Info("Starting consumer worker for topic {topic}", topic);
try
{
var topicDirectoryPath = Path.Combine(_directoryPath, topic);
var logDirectory = new LogDirectory(topicDirectoryPath, new Settings(logger: new KafkaesqueToToposLogger(_logger)));
var reader = logDirectory.GetReader();
while (!cancellationToken.IsCancellationRequested)
{
try
{
var resumePosition = _positionManager.Get(topic, 0).Result;
var(fileNumber, bytePosition) = resumePosition.ToKafkaesquePosition();
_logger.Debug("Resuming consumer from file {fileNumber} byte {bytePosition}", fileNumber, bytePosition);
foreach (var eventData in reader.Read(fileNumber, bytePosition, cancellationToken: cancellationToken))
{
var transportMessage = JsonConvert.DeserializeObject <TransportMessage>(Encoding.UTF8.GetString(eventData.Data));
var kafkaesqueEventPosition = new KafkaesquePosition(eventData.FileNumber, eventData.BytePosition);
var eventPosition = kafkaesqueEventPosition.ToPosition(topic, partition: 0);
var receivedTransportMessage = new ReceivedTransportMessage(eventPosition, transportMessage.Headers, transportMessage.Body);
_logger.Debug("Received event {position}", eventPosition);
_consumerDispatcher.Dispatch(receivedTransportMessage);
}
}
catch (Exception exception)
{
_logger.Warn(exception, "Error in consumer worker for topic {topic} - waiting 10 s", topic);
Task.Delay(TimeSpan.FromSeconds(10), cancellationToken)
.Wait(cancellationToken);
}
}
}
catch (OperationCanceledException) when(cancellationToken.IsCancellationRequested)
{
// we're done
}
catch (Exception exception)
{
_logger.Error(exception, "Unhandled exception in consumer worker for topic {topic}", topic);
}
finally
{
_logger.Info("Stopped consumer worker for topic {topic}", topic);
}
}
Lazy <LogWriter> CreateWriter(string topic)
{
var topicDirectoryPath = Path.Combine(_directoryPath, topic);
return(new Lazy <LogWriter>(() =>
{
var logDirectory = new LogDirectory(topicDirectoryPath, new Settings(logger: new KafkaesqueToToposLogger(_logger)));
_logger.Debug("Initializing new Kafkaesque writer with path {directoryPath}", topicDirectoryPath);
return logDirectory.GetWriter();
}));
}