コード例 #1
0
        void PumpTopic(string topic)
        {
            var cancellationToken = _cancellationTokenSource.Token;

            _logger.Info("Starting consumer worker for topic {topic}", topic);

            try
            {
                var topicDirectoryPath = Path.Combine(_directoryPath, topic);
                var logDirectory       = new LogDirectory(topicDirectoryPath, new Settings(logger: new KafkaesqueToToposLogger(_logger)));
                var reader             = logDirectory.GetReader();

                while (!cancellationToken.IsCancellationRequested)
                {
                    try
                    {
                        var resumePosition = _positionManager.Get(topic, 0).Result;

                        var(fileNumber, bytePosition) = resumePosition.ToKafkaesquePosition();

                        _logger.Debug("Resuming consumer from file {fileNumber} byte {bytePosition}", fileNumber, bytePosition);

                        foreach (var eventData in reader.Read(fileNumber, bytePosition, cancellationToken: cancellationToken))
                        {
                            var transportMessage         = JsonConvert.DeserializeObject <TransportMessage>(Encoding.UTF8.GetString(eventData.Data));
                            var kafkaesqueEventPosition  = new KafkaesquePosition(eventData.FileNumber, eventData.BytePosition);
                            var eventPosition            = kafkaesqueEventPosition.ToPosition(topic, partition: 0);
                            var receivedTransportMessage = new ReceivedTransportMessage(eventPosition, transportMessage.Headers, transportMessage.Body);

                            _logger.Debug("Received event {position}", eventPosition);

                            _consumerDispatcher.Dispatch(receivedTransportMessage);
                        }
                    }
                    catch (Exception exception)
                    {
                        _logger.Warn(exception, "Error in consumer worker for topic {topic} - waiting 10 s", topic);

                        Task.Delay(TimeSpan.FromSeconds(10), cancellationToken)
                        .Wait(cancellationToken);
                    }
                }
            }
            catch (OperationCanceledException) when(cancellationToken.IsCancellationRequested)
            {
                // we're done
            }
            catch (Exception exception)
            {
                _logger.Error(exception, "Unhandled exception in consumer worker for topic {topic}", topic);
            }
            finally
            {
                _logger.Info("Stopped consumer worker for topic {topic}", topic);
            }
        }
コード例 #2
0
        Lazy <LogWriter> CreateWriter(string topic)
        {
            var topicDirectoryPath = Path.Combine(_directoryPath, topic);

            return(new Lazy <LogWriter>(() =>
            {
                var logDirectory = new LogDirectory(topicDirectoryPath, new Settings(logger: new KafkaesqueToToposLogger(_logger)));

                _logger.Debug("Initializing new Kafkaesque writer with path {directoryPath}", topicDirectoryPath);

                return logDirectory.GetWriter();
            }));
        }