Example #1
0
        public void Init(IProviderConfiguration config, string providerName, Logger logger, IServiceProvider serviceProvider)
        {
            if (config == null)
            {
                throw new ArgumentNullException(nameof(config));
            }
            if (logger == null)
            {
                throw new ArgumentNullException(nameof(logger));
            }
            if (String.IsNullOrEmpty(providerName))
            {
                throw new ArgumentNullException(nameof(providerName));
            }

            // Creating an options object with all the config values
            _options = new KafkaStreamProviderOptions(config);

            if (!_options.UsingExternalMetrics)
            {
                Metric.Config.WithHttpEndpoint($"http://localhost:{_options.MetricsPort}/");
            }

            if (!_options.IncludeMetrics)
            {
                Metric.Context("KafkaStreamProvider").Advanced.CompletelyDisableMetrics();
            }

            _providerName      = providerName;
            _streamQueueMapper = new HashRingBasedStreamQueueMapper(_options.NumOfQueues, providerName);
            _logger            = logger;
            _adapter           = new KafkaQueueAdapter(_streamQueueMapper, _options, providerName, new KafkaBatchFactory(), _logger);
            _adapterCache      = new TimedQueueAdapterCache(this, TimeSpan.FromSeconds(_options.CacheTimespanInSeconds), _options.CacheSize, _options.CacheNumOfBuckets, logger);
        }
Example #2
0
        public KafkaQueueAdapterReceiver(QueueId queueId, IManualConsumer consumer, KafkaStreamProviderOptions options,
                                         IKafkaBatchFactory factory, ILogger logger)
        {
            // input checks
            if (queueId == null)
            {
                throw new ArgumentNullException(nameof(queueId));
            }
            if (consumer == null)
            {
                throw new ArgumentNullException(nameof(consumer));
            }
            if (factory == null)
            {
                throw new ArgumentNullException(nameof(factory));
            }
            if (options == null)
            {
                throw new ArgumentNullException(nameof(options));
            }
            if (logger == null)
            {
                throw new ArgumentNullException(nameof(logger));
            }

            _counterCurrentOffset = Metrics.Metric.Context("KafkaStreamProvider").Counter($"CurrentOffset queueId:({queueId.GetNumericId()})", unit:  Unit.Custom("Log"));

            _options  = options;
            Id        = queueId;
            _consumer = consumer;
            _factory  = factory;
            _logger   = logger;
        }
        public void Init(IProviderConfiguration config, string providerName, Logger logger, IServiceProvider serviceProvider)
        {
            if (config == null) throw new ArgumentNullException(nameof(config));
            if (logger == null) throw new ArgumentNullException(nameof(logger));
            if (String.IsNullOrEmpty(providerName)) throw new ArgumentNullException(nameof(providerName));

            // Creating an options object with all the config values
            _options = new KafkaStreamProviderOptions(config);

            if (!_options.UsingExternalMetrics)
            {
                Metric.Config.WithHttpEndpoint($"http://localhost:{_options.MetricsPort}/");
            }

            if (!_options.IncludeMetrics)
            {
                Metric.Context("KafkaStreamProvider").Advanced.CompletelyDisableMetrics();
            }

            _providerName = providerName;
            _streamQueueMapper = new HashRingBasedStreamQueueMapper(_options.NumOfQueues, providerName);
            _logger = logger;
            _adapter = new KafkaQueueAdapter(_streamQueueMapper, _options, providerName, new KafkaBatchFactory(), _logger);
            _adapterCache = new TimedQueueAdapterCache(this, TimeSpan.FromSeconds(_options.CacheTimespanInSeconds), _options.CacheSize, _options.CacheNumOfBuckets, logger);
        }
        public KafkaQueueAdapterReceiverUnitTests()
        {
            Mock<Logger> loggerMock = new Mock<Logger>();
            _logger = loggerMock.Object;

            var connectionStrings = new List<Uri> {new Uri("http://192.168.10.27:9092")};
            var topicName = "TestTopic";
            var consumerGroupName = "TestConsumerGroup";
            _options = new KafkaStreamProviderOptions(connectionStrings.ToArray(), topicName, consumerGroupName);
            _id = QueueId.GetQueueId("test", 0, 0);
        }
        public KafkaQueueAdapterUnitTests()
        {
            Mock<Logger> loggerMock = new Mock<Logger>();
            var connectionStrings = new List<Uri> {new Uri("http://kafka1:9092;http://kafka2:9092") };
            var topicName = "Jonathan.ab.KafkaStreamProviderTestsNew";
            var consumerGroupName = "TestConsumerGroupName";

            _logger = loggerMock.Object;
            _options = new KafkaStreamProviderOptions(connectionStrings.ToArray(), topicName, consumerGroupName);
            _streamQueueMapper = new HashRingBasedStreamQueueMapper(_options.NumOfQueues, _options.TopicName);
        }
        public KafkaQueueAdapter(HashRingBasedStreamQueueMapper queueMapper, KafkaStreamProviderOptions options,
                                 string providerName, IKafkaBatchFactory batchFactory, ILogger logger)
        {
            if (options == null)
            {
                throw new ArgumentNullException(nameof(options));
            }
            if (batchFactory == null)
            {
                throw new ArgumentNullException(nameof(batchFactory));
            }
            if (queueMapper == null)
            {
                throw new ArgumentNullException(nameof(queueMapper));
            }
            if (logger == null)
            {
                throw new ArgumentNullException(nameof(logger));
            }
            if (string.IsNullOrEmpty(providerName))
            {
                throw new ArgumentNullException(nameof(providerName));
            }

            _options           = options;
            _streamQueueMapper = queueMapper;
            Name          = providerName;
            _batchFactory = batchFactory;
            _logger       = logger;

            // Creating a producer
            KafkaOptions kafkaOptions = new KafkaOptions(_options.ConnectionStrings.ToArray())
            {
                Log = new KafkaLogBridge(logger)
            };
            var broker = new BrokerRouter(kafkaOptions);

            _producer = new Producer(broker)
            {
                BatchDelayTime = TimeSpan.FromMilliseconds(_options.TimeToWaitForBatchInMs),
                BatchSize      = _options.ProduceBatchSize
            };
            _gateway = new ProtocolGateway(kafkaOptions);

            _logger.Info("KafkaQueueAdapter - Created a KafkaQueueAdapter");
        }
        public KafkaQueueAdapterReceiver(QueueId queueId, IManualConsumer consumer, KafkaStreamProviderOptions options,
            IKafkaBatchFactory factory, Logger logger)
        {
            // input checks
            if (queueId == null) throw new ArgumentNullException(nameof(queueId));
            if (consumer == null) throw new ArgumentNullException(nameof(consumer));
            if (factory == null) throw new ArgumentNullException(nameof(factory));
            if (options == null) throw new ArgumentNullException(nameof(options));
            if (logger == null) throw new ArgumentNullException(nameof(logger));

            _counterCurrentOffset = Metric.Context("KafkaStreamProvider").Counter($"CurrentOffset queueId:({queueId.GetNumericId()})", unit:  Unit.Custom("Log"));
       
            _options = options;
            Id = queueId;
            _consumer = consumer;
            _factory = factory;
            _logger = logger;
        }
        public async Task GetMessagesQueueSavingBatchAfterFewFetchesTest()
        {
            bool hasCommited = false;

            Message message1 = new Message() { Value = new byte[] { 1, 2, 3, 4 }, Meta = new MessageMetadata() { Offset = 0, PartitionId = 0 } };
            Message message2 = new Message() { Value = new byte[] { 4, 5, 6, 7 }, Meta = new MessageMetadata() { Offset = 1, PartitionId = 0 } };
            Message message3 = new Message() { Value = new byte[] { 8, 9, 10, 11 }, Meta = new MessageMetadata() { Offset = 2, PartitionId = 0 } };
            Message message4 = new Message() { Value = new byte[] { 12, 13, 14, 15 }, Meta = new MessageMetadata() { Offset = 3, PartitionId = 0 } };
            Message message5 = new Message() { Value = new byte[] { 16, 17, 18, 19 }, Meta = new MessageMetadata() { Offset = 4, PartitionId = 0 } };
            Message message6 = new Message() { Value = new byte[] { 21, 22, 23, 24 }, Meta = new MessageMetadata() { Offset = 5, PartitionId = 0 } };

            Mock<IManualConsumer> consumerMock = new Mock<IManualConsumer>();
            Mock<IBatchContainer> batchContainerMock1 = new Mock<IBatchContainer>();
            Mock<IBatchContainer> batchContainerMock2 = new Mock<IBatchContainer>();
            Mock<IBatchContainer> batchContainerMock3 = new Mock<IBatchContainer>();
            Mock<IBatchContainer> batchContainerMock4 = new Mock<IBatchContainer>();
            Mock<IBatchContainer> batchContainerMock5 = new Mock<IBatchContainer>();
            Mock<IBatchContainer> batchContainerMock6 = new Mock<IBatchContainer>();

            var returnCollection1 = Task.FromResult<IEnumerable<Message>>(new List<Message>() { message1 });
            consumerMock.Setup(x => x.FetchMessages(6, 0)).Returns(returnCollection1);

            var returnCollection2 = Task.FromResult<IEnumerable<Message>>(new List<Message>() { message2, message3});
            consumerMock.Setup(x => x.FetchMessages(6, 1)).Returns(returnCollection2);

            var returnCollection3 = Task.FromResult<IEnumerable<Message>>(new List<Message>() { message4, message5, message6 });
            consumerMock.Setup(x => x.FetchMessages(6, 3)).Returns(returnCollection3);

            consumerMock.Setup(x => x.FetchOffset(_options.ConsumerGroupName)).Returns(Task.FromResult<long>(0));

            consumerMock.Setup(x => x.UpdateOrCreateOffset(_options.ConsumerGroupName, 3)).Returns(TaskDone.Done)
                .Callback(() => hasCommited = true);

            Mock<IKafkaBatchFactory> factoryMock = new Mock<IKafkaBatchFactory>();

            factoryMock.Setup(x => x.FromKafkaMessage(message1, It.IsAny<long>()))
                .Returns(batchContainerMock1.Object);
            factoryMock.Setup(x => x.FromKafkaMessage(message2, It.IsAny<long>()))
                .Returns(batchContainerMock2.Object);
            factoryMock.Setup(x => x.FromKafkaMessage(message3, It.IsAny<long>()))
                .Returns(batchContainerMock3.Object);
            factoryMock.Setup(x => x.FromKafkaMessage(message4, It.IsAny<long>()))
            .Returns(batchContainerMock4.Object);
            factoryMock.Setup(x => x.FromKafkaMessage(message5, It.IsAny<long>()))
                .Returns(batchContainerMock5.Object);
            factoryMock.Setup(x => x.FromKafkaMessage(message6, It.IsAny<long>()))
                .Returns(batchContainerMock6.Object);

            var newOptions = new KafkaStreamProviderOptions(_options.ConnectionStrings, _options.TopicName, _options.ConsumerGroupName){OffsetCommitInterval = 2, ShouldInitWithLastOffset = false};

            KafkaQueueAdapterReceiver adapterReceiver = new KafkaQueueAdapterReceiver(_id, consumerMock.Object, newOptions, factoryMock.Object, _logger);

            await adapterReceiver.Initialize(TimeSpan.MaxValue);

            int numToTake = 6;

            // Asking once, should return 1 message
            var result = await adapterReceiver.GetQueueMessagesAsync(6);

            Assert.AreEqual(1, result.Count);
            Assert.AreEqual(batchContainerMock1.Object, result[0]);

            // Now sth has happened, restating the adapter, the offset should not be saved
            adapterReceiver = new KafkaQueueAdapterReceiver(_id, consumerMock.Object, newOptions, factoryMock.Object, _logger);
            await adapterReceiver.Initialize(TimeSpan.MaxValue);

            // Asking once again, should return 1 message
            result = await adapterReceiver.GetQueueMessagesAsync(6);

            Assert.AreEqual(1, result.Count);
            Assert.AreEqual(batchContainerMock1.Object, result[0]);

            // Asking second again, should return 2 messages and commit
            result = await adapterReceiver.GetQueueMessagesAsync(6);

            Assert.AreEqual(2, result.Count);
            Assert.AreEqual(batchContainerMock2.Object, result[0]);
            Assert.AreEqual(batchContainerMock3.Object, result[1]);

            if (hasCommited)
            {
                consumerMock.Setup(x => x.FetchOffset(_options.ConsumerGroupName)).Returns(Task.FromResult<long>(3));
            }

            // Now sth happened, but this time the offset was committed
            adapterReceiver = new KafkaQueueAdapterReceiver(_id, consumerMock.Object, newOptions, factoryMock.Object, _logger);
            await adapterReceiver.Initialize(TimeSpan.MaxValue);

            // This request should return the last 3 messages
            result = await adapterReceiver.GetQueueMessagesAsync(numToTake);

            Assert.AreEqual(3, result.Count);
            Assert.AreEqual(batchContainerMock4.Object, result[0]);
            Assert.AreEqual(batchContainerMock5.Object, result[1]);
            Assert.AreEqual(batchContainerMock6.Object, result[2]);
        }
        public async Task InitializeConsumerGroupDoesntExistRecoveryTest()
        {
            var wantedOffset = 10;
            var savedNewOffset = false;

            Mock<IManualConsumer> consumerMock = new Mock<IManualConsumer>();
            Mock<IKafkaBatchFactory> factoryMock = new Mock<IKafkaBatchFactory>();
            KafkaStreamProviderOptions mockOptions = new KafkaStreamProviderOptions(_options.ConnectionStrings,
                _options.TopicName, _options.ConsumerGroupName) {ShouldInitWithLastOffset = false};

            consumerMock.Setup(x => x.FetchOffset(_options.ConsumerGroupName)).Throws(new KafkaApplicationException("Test") { ErrorCode = (int)ErrorResponseCode.UnknownTopicOrPartition });

            KafkaQueueAdapterReceiver adapterReceiver = new KafkaQueueAdapterReceiver(_id, consumerMock.Object, _options, factoryMock.Object, _logger);
            consumerMock.Setup(x => x.FetchLastOffset()).ReturnsAsync(wantedOffset).Callback(() => savedNewOffset=true);            

            await adapterReceiver.Initialize(TimeSpan.MaxValue);

            if (savedNewOffset)
            {
                consumerMock.Setup(x => x.FetchOffset(_options.ConsumerGroupName)).ReturnsAsync(wantedOffset);    
            }

            await adapterReceiver.Initialize(TimeSpan.MaxValue);

            Assert.AreEqual(wantedOffset, adapterReceiver.CurrentOffset);
        }
        public async Task InitializeGetOffsetFromConsumerGroupTest()
        {
            var wantedOffset = 10;

            Mock<IManualConsumer> consumerMock = new Mock<IManualConsumer>();
            Mock<IKafkaBatchFactory> factoryMock = new Mock<IKafkaBatchFactory>();
            KafkaStreamProviderOptions mockOptions = new KafkaStreamProviderOptions(_options.ConnectionStrings,
                _options.TopicName, _options.ConsumerGroupName) {ShouldInitWithLastOffset = false};

            consumerMock.Setup(x => x.FetchOffset(_options.ConsumerGroupName)).ReturnsAsync(wantedOffset);

            KafkaQueueAdapterReceiver adapterReceiver = new KafkaQueueAdapterReceiver(_id, consumerMock.Object, mockOptions, factoryMock.Object, _logger);

            await adapterReceiver.Initialize(TimeSpan.MaxValue);

            Assert.AreEqual(wantedOffset, adapterReceiver.CurrentOffset);
        }
        public void QueueMessageBatchAsyncQueueingTwiceDifferentQueuesTest()
        {
            Mock<IKafkaBatchFactory> factoryMock = new Mock<IKafkaBatchFactory>();
            Dictionary<string, object> requestContext = new Dictionary<string, object>();

            factoryMock.Setup(x => x.ToKafkaMessage(It.IsAny<Guid>(), It.IsAny<string>(), It.IsAny<IEnumerable<int>>(), requestContext)).Returns(new Message() { Value = new byte[] { 0, 1, 2, 3 } });

            var twoQueuesStreamMapper = new HashRingBasedStreamQueueMapper(2, _options.TopicName);
            var twoQueuesOptions = new KafkaStreamProviderOptions(_options.ConnectionStrings, _options.TopicName, _options.ConsumerGroupName){NumOfQueues = 2};

            KafkaQueueAdapter adapter = new KafkaQueueAdapter(_streamQueueMapper, _options, _providerName, factoryMock.Object, _logger);            

            var first = Guid.NewGuid();
            var second = Guid.NewGuid();

            // Becuase we cannot mock the queue mapper.. we need to make sure we two guids that will return different queues...
            bool willGiveDifferentQueue = !(twoQueuesStreamMapper.GetQueueForStream(first, "test").Equals(twoQueuesStreamMapper.GetQueueForStream(second, "otherTest")));
            while (!willGiveDifferentQueue)
            {
                second = Guid.NewGuid();
                willGiveDifferentQueue = !(twoQueuesStreamMapper.GetQueueForStream(first, "test").Equals(twoQueuesStreamMapper.GetQueueForStream(second, "otherTest")));
            }

            Task.WaitAll(adapter.QueueMessageBatchAsync(first, "test", new List<int>() { 1, 2, 3, 4 }, null, requestContext),
                         adapter.QueueMessageBatchAsync(second, "otherTest", new List<int>() { 1, 2, 3, 4 }, null, requestContext));
        }
        public async Task QueueMessageBatchAsyncAllNoAck()
        {
            Mock<IKafkaBatchFactory> factoryMock = new Mock<IKafkaBatchFactory>();
            Dictionary<string, object> requestContext = new Dictionary<string, object>();

            factoryMock.Setup(x => x.ToKafkaMessage(It.IsAny<Guid>(), It.IsAny<string>(), It.IsAny<IEnumerable<int>>(), requestContext)).Returns((Message)null);

            KafkaStreamProviderOptions differentOptions = new KafkaStreamProviderOptions(_options.ConnectionStrings,
                _options.TopicName, _options.ConsumerGroupName) {AckLevel = 0};

            KafkaQueueAdapter adapter = new KafkaQueueAdapter(_streamQueueMapper, differentOptions, _providerName, factoryMock.Object, _logger);

            Guid myGuid = Guid.NewGuid();

            await adapter.QueueMessageBatchAsync(myGuid, "Test", new List<int>() { 1, 2, 3, 4 }, null, requestContext);
        }