public static IEnumerable <TopicPartitionOffset> PartitionsAssigned <T1, T2>(ILogger logger, IConsumer <T1, T2> consumer, IEnumerable <TopicPartition> partitions, IPositionManager positionManager)
{
var partitionsList = partitions.ToList();
if (!partitionsList.Any())
{
return(Enumerable.Empty <TopicPartitionOffset>());
}
var partitionsByTopic = partitionsList
.GroupBy(p => p.Topic)
.Select(g => new { Topic = g.Key, Partitions = g.Select(p => p.Partition.Value) })
.ToList();
logger.Info("Assignment: {@partitions}", partitionsByTopic);
return(partitionsList
.Select(tp => new
{
TopicPartition = tp,
Position = AsyncHelpers.GetAsync(() => positionManager.Get(tp.Topic, tp.Partition.Value))
})
.Select(a => a.Position?.Advance(1).ToTopicPartitionOffset() // either resume from the event following the last one successfully committedf
?? a.TopicPartition.WithOffset(Offset.Beginning))); // or just resume from the beginning
}
void PumpTopic(string topic)
{
var cancellationToken = _cancellationTokenSource.Token;
_logger.Info("Starting consumer worker for topic {topic}", topic);
try
{
var topicDirectoryPath = Path.Combine(_directoryPath, topic);
var logDirectory = new LogDirectory(topicDirectoryPath, new Settings(logger: new KafkaesqueToToposLogger(_logger)));
var reader = logDirectory.GetReader();
while (!cancellationToken.IsCancellationRequested)
{
try
{
var resumePosition = _positionManager.Get(topic, 0).Result;
var(fileNumber, bytePosition) = resumePosition.ToKafkaesquePosition();
_logger.Debug("Resuming consumer from file {fileNumber} byte {bytePosition}", fileNumber, bytePosition);
foreach (var eventData in reader.Read(fileNumber, bytePosition, cancellationToken: cancellationToken))
{
var transportMessage = JsonConvert.DeserializeObject <TransportMessage>(Encoding.UTF8.GetString(eventData.Data));
var kafkaesqueEventPosition = new KafkaesquePosition(eventData.FileNumber, eventData.BytePosition);
var eventPosition = kafkaesqueEventPosition.ToPosition(topic, partition: 0);
var receivedTransportMessage = new ReceivedTransportMessage(eventPosition, transportMessage.Headers, transportMessage.Body);
_logger.Debug("Received event {position}", eventPosition);
_consumerDispatcher.Dispatch(receivedTransportMessage);
}
}
catch (Exception exception)
{
_logger.Warn(exception, "Error in consumer worker for topic {topic} - waiting 10 s", topic);
Task.Delay(TimeSpan.FromSeconds(10), cancellationToken)
.Wait(cancellationToken);
}
}
}
catch (OperationCanceledException) when(cancellationToken.IsCancellationRequested)
{
// we're done
}
catch (Exception exception)
{
_logger.Error(exception, "Unhandled exception in consumer worker for topic {topic}", topic);
}
finally
{
_logger.Info("Stopped consumer worker for topic {topic}", topic);
}
}
public async Task <Position> Get(string topic, int partition)
{
var position = await _positionManager.Get(topic, partition);
return((isDefault : position.IsDefault, startFrom : _startFromPosition) switch
{
(isDefault : true, startFrom : StartFromPosition.Beginning) => Position.Default(topic, partition),
(isDefault : true, startFrom : StartFromPosition.Now) => Position.OnlyNew(topic, partition),
_ => position
});
public static IEnumerable <TopicPartitionOffset> PartitionsAssigned(
ILogger logger,
IReadOnlyList <TopicPartition> partitions,
IPositionManager positionManager,
Func <ConsumerContext, IEnumerable <TopicPartition>, Task> partitionsAssignedHandler,
ConsumerContext context
)
{
if (!partitions.Any())
{
return(Enumerable.Empty <TopicPartitionOffset>());
}
var partitionsByTopic = partitions
.GroupBy(p => p.Topic)
.Select(g => new { Topic = g.Key, Partitions = g.Select(p => p.Partition.Value) })
.ToList();
logger.Info("Assignment: {@partitions}", partitionsByTopic);
if (partitionsAssignedHandler != null)
{
AsyncHelpers.RunSync(() => partitionsAssignedHandler(context, partitions));
}
return(AsyncHelpers.GetAsync(async() =>
{
var results = await partitions
.Select(async tp => new
{
TopicPartition = tp,
Position = await positionManager.Get(tp.Topic, tp.Partition.Value)
})
.ToListAsync();
return results
.Select(a =>
{
if (a.Position.IsDefault)
{
return a.TopicPartition.WithOffset(Offset.Beginning);
}
if (a.Position.IsOnlyNew)
{
return a.TopicPartition.WithOffset(Offset.End);
}
return a.Position.Advance(1).ToTopicPartitionOffset();
})
.ToList();
}));
}