public static IEnumerable <TopicPartitionOffset> PartitionsAssigned <T1, T2>(ILogger logger, IConsumer <T1, T2> consumer, IEnumerable <TopicPartition> partitions, IPositionManager positionManager) { var partitionsList = partitions.ToList(); if (!partitionsList.Any()) { return(Enumerable.Empty <TopicPartitionOffset>()); } var partitionsByTopic = partitionsList .GroupBy(p => p.Topic) .Select(g => new { Topic = g.Key, Partitions = g.Select(p => p.Partition.Value) }) .ToList(); logger.Info("Assignment: {@partitions}", partitionsByTopic); return(partitionsList .Select(tp => new { TopicPartition = tp, Position = AsyncHelpers.GetAsync(() => positionManager.Get(tp.Topic, tp.Partition.Value)) }) .Select(a => a.Position?.Advance(1).ToTopicPartitionOffset() // either resume from the event following the last one successfully committedf ?? a.TopicPartition.WithOffset(Offset.Beginning))); // or just resume from the beginning }
void PumpTopic(string topic) { var cancellationToken = _cancellationTokenSource.Token; _logger.Info("Starting consumer worker for topic {topic}", topic); try { var topicDirectoryPath = Path.Combine(_directoryPath, topic); var logDirectory = new LogDirectory(topicDirectoryPath, new Settings(logger: new KafkaesqueToToposLogger(_logger))); var reader = logDirectory.GetReader(); while (!cancellationToken.IsCancellationRequested) { try { var resumePosition = _positionManager.Get(topic, 0).Result; var(fileNumber, bytePosition) = resumePosition.ToKafkaesquePosition(); _logger.Debug("Resuming consumer from file {fileNumber} byte {bytePosition}", fileNumber, bytePosition); foreach (var eventData in reader.Read(fileNumber, bytePosition, cancellationToken: cancellationToken)) { var transportMessage = JsonConvert.DeserializeObject <TransportMessage>(Encoding.UTF8.GetString(eventData.Data)); var kafkaesqueEventPosition = new KafkaesquePosition(eventData.FileNumber, eventData.BytePosition); var eventPosition = kafkaesqueEventPosition.ToPosition(topic, partition: 0); var receivedTransportMessage = new ReceivedTransportMessage(eventPosition, transportMessage.Headers, transportMessage.Body); _logger.Debug("Received event {position}", eventPosition); _consumerDispatcher.Dispatch(receivedTransportMessage); } } catch (Exception exception) { _logger.Warn(exception, "Error in consumer worker for topic {topic} - waiting 10 s", topic); Task.Delay(TimeSpan.FromSeconds(10), cancellationToken) .Wait(cancellationToken); } } } catch (OperationCanceledException) when(cancellationToken.IsCancellationRequested) { // we're done } catch (Exception exception) { _logger.Error(exception, "Unhandled exception in consumer worker for topic {topic}", topic); } finally { _logger.Info("Stopped consumer worker for topic {topic}", topic); } }
public async Task <Position> Get(string topic, int partition) { var position = await _positionManager.Get(topic, partition); return((isDefault : position.IsDefault, startFrom : _startFromPosition) switch { (isDefault : true, startFrom : StartFromPosition.Beginning) => Position.Default(topic, partition), (isDefault : true, startFrom : StartFromPosition.Now) => Position.OnlyNew(topic, partition), _ => position });
public static IEnumerable <TopicPartitionOffset> PartitionsAssigned( ILogger logger, IReadOnlyList <TopicPartition> partitions, IPositionManager positionManager, Func <ConsumerContext, IEnumerable <TopicPartition>, Task> partitionsAssignedHandler, ConsumerContext context ) { if (!partitions.Any()) { return(Enumerable.Empty <TopicPartitionOffset>()); } var partitionsByTopic = partitions .GroupBy(p => p.Topic) .Select(g => new { Topic = g.Key, Partitions = g.Select(p => p.Partition.Value) }) .ToList(); logger.Info("Assignment: {@partitions}", partitionsByTopic); if (partitionsAssignedHandler != null) { AsyncHelpers.RunSync(() => partitionsAssignedHandler(context, partitions)); } return(AsyncHelpers.GetAsync(async() => { var results = await partitions .Select(async tp => new { TopicPartition = tp, Position = await positionManager.Get(tp.Topic, tp.Partition.Value) }) .ToListAsync(); return results .Select(a => { if (a.Position.IsDefault) { return a.TopicPartition.WithOffset(Offset.Beginning); } if (a.Position.IsOnlyNew) { return a.TopicPartition.WithOffset(Offset.End); } return a.Position.Advance(1).ToTopicPartitionOffset(); }) .ToList(); })); }