public void Init(IProviderConfiguration config, string providerName, Logger logger, IServiceProvider serviceProvider) { if (config == null) { throw new ArgumentNullException(nameof(config)); } if (logger == null) { throw new ArgumentNullException(nameof(logger)); } if (String.IsNullOrEmpty(providerName)) { throw new ArgumentNullException(nameof(providerName)); } // Creating an options object with all the config values _options = new KafkaStreamProviderOptions(config); if (!_options.UsingExternalMetrics) { Metric.Config.WithHttpEndpoint($"http://localhost:{_options.MetricsPort}/"); } if (!_options.IncludeMetrics) { Metric.Context("KafkaStreamProvider").Advanced.CompletelyDisableMetrics(); } _providerName = providerName; _streamQueueMapper = new HashRingBasedStreamQueueMapper(_options.NumOfQueues, providerName); _logger = logger; _adapter = new KafkaQueueAdapter(_streamQueueMapper, _options, providerName, new KafkaBatchFactory(), _logger); _adapterCache = new TimedQueueAdapterCache(this, TimeSpan.FromSeconds(_options.CacheTimespanInSeconds), _options.CacheSize, _options.CacheNumOfBuckets, logger); }
public KafkaQueueAdapterReceiver(QueueId queueId, IManualConsumer consumer, KafkaStreamProviderOptions options, IKafkaBatchFactory factory, ILogger logger) { // input checks if (queueId == null) { throw new ArgumentNullException(nameof(queueId)); } if (consumer == null) { throw new ArgumentNullException(nameof(consumer)); } if (factory == null) { throw new ArgumentNullException(nameof(factory)); } if (options == null) { throw new ArgumentNullException(nameof(options)); } if (logger == null) { throw new ArgumentNullException(nameof(logger)); } _counterCurrentOffset = Metrics.Metric.Context("KafkaStreamProvider").Counter($"CurrentOffset queueId:({queueId.GetNumericId()})", unit: Unit.Custom("Log")); _options = options; Id = queueId; _consumer = consumer; _factory = factory; _logger = logger; }
public void Init(IProviderConfiguration config, string providerName, Logger logger, IServiceProvider serviceProvider) { if (config == null) throw new ArgumentNullException(nameof(config)); if (logger == null) throw new ArgumentNullException(nameof(logger)); if (String.IsNullOrEmpty(providerName)) throw new ArgumentNullException(nameof(providerName)); // Creating an options object with all the config values _options = new KafkaStreamProviderOptions(config); if (!_options.UsingExternalMetrics) { Metric.Config.WithHttpEndpoint($"http://localhost:{_options.MetricsPort}/"); } if (!_options.IncludeMetrics) { Metric.Context("KafkaStreamProvider").Advanced.CompletelyDisableMetrics(); } _providerName = providerName; _streamQueueMapper = new HashRingBasedStreamQueueMapper(_options.NumOfQueues, providerName); _logger = logger; _adapter = new KafkaQueueAdapter(_streamQueueMapper, _options, providerName, new KafkaBatchFactory(), _logger); _adapterCache = new TimedQueueAdapterCache(this, TimeSpan.FromSeconds(_options.CacheTimespanInSeconds), _options.CacheSize, _options.CacheNumOfBuckets, logger); }
public KafkaQueueAdapterReceiverUnitTests() { Mock<Logger> loggerMock = new Mock<Logger>(); _logger = loggerMock.Object; var connectionStrings = new List<Uri> {new Uri("http://192.168.10.27:9092")}; var topicName = "TestTopic"; var consumerGroupName = "TestConsumerGroup"; _options = new KafkaStreamProviderOptions(connectionStrings.ToArray(), topicName, consumerGroupName); _id = QueueId.GetQueueId("test", 0, 0); }
public KafkaQueueAdapterUnitTests() { Mock<Logger> loggerMock = new Mock<Logger>(); var connectionStrings = new List<Uri> {new Uri("http://kafka1:9092;http://kafka2:9092") }; var topicName = "Jonathan.ab.KafkaStreamProviderTestsNew"; var consumerGroupName = "TestConsumerGroupName"; _logger = loggerMock.Object; _options = new KafkaStreamProviderOptions(connectionStrings.ToArray(), topicName, consumerGroupName); _streamQueueMapper = new HashRingBasedStreamQueueMapper(_options.NumOfQueues, _options.TopicName); }
public KafkaQueueAdapter(HashRingBasedStreamQueueMapper queueMapper, KafkaStreamProviderOptions options, string providerName, IKafkaBatchFactory batchFactory, ILogger logger) { if (options == null) { throw new ArgumentNullException(nameof(options)); } if (batchFactory == null) { throw new ArgumentNullException(nameof(batchFactory)); } if (queueMapper == null) { throw new ArgumentNullException(nameof(queueMapper)); } if (logger == null) { throw new ArgumentNullException(nameof(logger)); } if (string.IsNullOrEmpty(providerName)) { throw new ArgumentNullException(nameof(providerName)); } _options = options; _streamQueueMapper = queueMapper; Name = providerName; _batchFactory = batchFactory; _logger = logger; // Creating a producer KafkaOptions kafkaOptions = new KafkaOptions(_options.ConnectionStrings.ToArray()) { Log = new KafkaLogBridge(logger) }; var broker = new BrokerRouter(kafkaOptions); _producer = new Producer(broker) { BatchDelayTime = TimeSpan.FromMilliseconds(_options.TimeToWaitForBatchInMs), BatchSize = _options.ProduceBatchSize }; _gateway = new ProtocolGateway(kafkaOptions); _logger.Info("KafkaQueueAdapter - Created a KafkaQueueAdapter"); }
public KafkaQueueAdapterReceiver(QueueId queueId, IManualConsumer consumer, KafkaStreamProviderOptions options, IKafkaBatchFactory factory, Logger logger) { // input checks if (queueId == null) throw new ArgumentNullException(nameof(queueId)); if (consumer == null) throw new ArgumentNullException(nameof(consumer)); if (factory == null) throw new ArgumentNullException(nameof(factory)); if (options == null) throw new ArgumentNullException(nameof(options)); if (logger == null) throw new ArgumentNullException(nameof(logger)); _counterCurrentOffset = Metric.Context("KafkaStreamProvider").Counter($"CurrentOffset queueId:({queueId.GetNumericId()})", unit: Unit.Custom("Log")); _options = options; Id = queueId; _consumer = consumer; _factory = factory; _logger = logger; }
public async Task GetMessagesQueueSavingBatchAfterFewFetchesTest() { bool hasCommited = false; Message message1 = new Message() { Value = new byte[] { 1, 2, 3, 4 }, Meta = new MessageMetadata() { Offset = 0, PartitionId = 0 } }; Message message2 = new Message() { Value = new byte[] { 4, 5, 6, 7 }, Meta = new MessageMetadata() { Offset = 1, PartitionId = 0 } }; Message message3 = new Message() { Value = new byte[] { 8, 9, 10, 11 }, Meta = new MessageMetadata() { Offset = 2, PartitionId = 0 } }; Message message4 = new Message() { Value = new byte[] { 12, 13, 14, 15 }, Meta = new MessageMetadata() { Offset = 3, PartitionId = 0 } }; Message message5 = new Message() { Value = new byte[] { 16, 17, 18, 19 }, Meta = new MessageMetadata() { Offset = 4, PartitionId = 0 } }; Message message6 = new Message() { Value = new byte[] { 21, 22, 23, 24 }, Meta = new MessageMetadata() { Offset = 5, PartitionId = 0 } }; Mock<IManualConsumer> consumerMock = new Mock<IManualConsumer>(); Mock<IBatchContainer> batchContainerMock1 = new Mock<IBatchContainer>(); Mock<IBatchContainer> batchContainerMock2 = new Mock<IBatchContainer>(); Mock<IBatchContainer> batchContainerMock3 = new Mock<IBatchContainer>(); Mock<IBatchContainer> batchContainerMock4 = new Mock<IBatchContainer>(); Mock<IBatchContainer> batchContainerMock5 = new Mock<IBatchContainer>(); Mock<IBatchContainer> batchContainerMock6 = new Mock<IBatchContainer>(); var returnCollection1 = Task.FromResult<IEnumerable<Message>>(new List<Message>() { message1 }); consumerMock.Setup(x => x.FetchMessages(6, 0)).Returns(returnCollection1); var returnCollection2 = Task.FromResult<IEnumerable<Message>>(new List<Message>() { message2, message3}); consumerMock.Setup(x => x.FetchMessages(6, 1)).Returns(returnCollection2); var returnCollection3 = Task.FromResult<IEnumerable<Message>>(new List<Message>() { message4, message5, message6 }); consumerMock.Setup(x => x.FetchMessages(6, 3)).Returns(returnCollection3); consumerMock.Setup(x => x.FetchOffset(_options.ConsumerGroupName)).Returns(Task.FromResult<long>(0)); consumerMock.Setup(x => x.UpdateOrCreateOffset(_options.ConsumerGroupName, 3)).Returns(TaskDone.Done) .Callback(() => hasCommited = true); Mock<IKafkaBatchFactory> factoryMock = new Mock<IKafkaBatchFactory>(); factoryMock.Setup(x => x.FromKafkaMessage(message1, It.IsAny<long>())) .Returns(batchContainerMock1.Object); factoryMock.Setup(x => x.FromKafkaMessage(message2, It.IsAny<long>())) .Returns(batchContainerMock2.Object); factoryMock.Setup(x => x.FromKafkaMessage(message3, It.IsAny<long>())) .Returns(batchContainerMock3.Object); factoryMock.Setup(x => x.FromKafkaMessage(message4, It.IsAny<long>())) .Returns(batchContainerMock4.Object); factoryMock.Setup(x => x.FromKafkaMessage(message5, It.IsAny<long>())) .Returns(batchContainerMock5.Object); factoryMock.Setup(x => x.FromKafkaMessage(message6, It.IsAny<long>())) .Returns(batchContainerMock6.Object); var newOptions = new KafkaStreamProviderOptions(_options.ConnectionStrings, _options.TopicName, _options.ConsumerGroupName){OffsetCommitInterval = 2, ShouldInitWithLastOffset = false}; KafkaQueueAdapterReceiver adapterReceiver = new KafkaQueueAdapterReceiver(_id, consumerMock.Object, newOptions, factoryMock.Object, _logger); await adapterReceiver.Initialize(TimeSpan.MaxValue); int numToTake = 6; // Asking once, should return 1 message var result = await adapterReceiver.GetQueueMessagesAsync(6); Assert.AreEqual(1, result.Count); Assert.AreEqual(batchContainerMock1.Object, result[0]); // Now sth has happened, restating the adapter, the offset should not be saved adapterReceiver = new KafkaQueueAdapterReceiver(_id, consumerMock.Object, newOptions, factoryMock.Object, _logger); await adapterReceiver.Initialize(TimeSpan.MaxValue); // Asking once again, should return 1 message result = await adapterReceiver.GetQueueMessagesAsync(6); Assert.AreEqual(1, result.Count); Assert.AreEqual(batchContainerMock1.Object, result[0]); // Asking second again, should return 2 messages and commit result = await adapterReceiver.GetQueueMessagesAsync(6); Assert.AreEqual(2, result.Count); Assert.AreEqual(batchContainerMock2.Object, result[0]); Assert.AreEqual(batchContainerMock3.Object, result[1]); if (hasCommited) { consumerMock.Setup(x => x.FetchOffset(_options.ConsumerGroupName)).Returns(Task.FromResult<long>(3)); } // Now sth happened, but this time the offset was committed adapterReceiver = new KafkaQueueAdapterReceiver(_id, consumerMock.Object, newOptions, factoryMock.Object, _logger); await adapterReceiver.Initialize(TimeSpan.MaxValue); // This request should return the last 3 messages result = await adapterReceiver.GetQueueMessagesAsync(numToTake); Assert.AreEqual(3, result.Count); Assert.AreEqual(batchContainerMock4.Object, result[0]); Assert.AreEqual(batchContainerMock5.Object, result[1]); Assert.AreEqual(batchContainerMock6.Object, result[2]); }
public async Task InitializeConsumerGroupDoesntExistRecoveryTest() { var wantedOffset = 10; var savedNewOffset = false; Mock<IManualConsumer> consumerMock = new Mock<IManualConsumer>(); Mock<IKafkaBatchFactory> factoryMock = new Mock<IKafkaBatchFactory>(); KafkaStreamProviderOptions mockOptions = new KafkaStreamProviderOptions(_options.ConnectionStrings, _options.TopicName, _options.ConsumerGroupName) {ShouldInitWithLastOffset = false}; consumerMock.Setup(x => x.FetchOffset(_options.ConsumerGroupName)).Throws(new KafkaApplicationException("Test") { ErrorCode = (int)ErrorResponseCode.UnknownTopicOrPartition }); KafkaQueueAdapterReceiver adapterReceiver = new KafkaQueueAdapterReceiver(_id, consumerMock.Object, _options, factoryMock.Object, _logger); consumerMock.Setup(x => x.FetchLastOffset()).ReturnsAsync(wantedOffset).Callback(() => savedNewOffset=true); await adapterReceiver.Initialize(TimeSpan.MaxValue); if (savedNewOffset) { consumerMock.Setup(x => x.FetchOffset(_options.ConsumerGroupName)).ReturnsAsync(wantedOffset); } await adapterReceiver.Initialize(TimeSpan.MaxValue); Assert.AreEqual(wantedOffset, adapterReceiver.CurrentOffset); }
public async Task InitializeGetOffsetFromConsumerGroupTest() { var wantedOffset = 10; Mock<IManualConsumer> consumerMock = new Mock<IManualConsumer>(); Mock<IKafkaBatchFactory> factoryMock = new Mock<IKafkaBatchFactory>(); KafkaStreamProviderOptions mockOptions = new KafkaStreamProviderOptions(_options.ConnectionStrings, _options.TopicName, _options.ConsumerGroupName) {ShouldInitWithLastOffset = false}; consumerMock.Setup(x => x.FetchOffset(_options.ConsumerGroupName)).ReturnsAsync(wantedOffset); KafkaQueueAdapterReceiver adapterReceiver = new KafkaQueueAdapterReceiver(_id, consumerMock.Object, mockOptions, factoryMock.Object, _logger); await adapterReceiver.Initialize(TimeSpan.MaxValue); Assert.AreEqual(wantedOffset, adapterReceiver.CurrentOffset); }
public void QueueMessageBatchAsyncQueueingTwiceDifferentQueuesTest() { Mock<IKafkaBatchFactory> factoryMock = new Mock<IKafkaBatchFactory>(); Dictionary<string, object> requestContext = new Dictionary<string, object>(); factoryMock.Setup(x => x.ToKafkaMessage(It.IsAny<Guid>(), It.IsAny<string>(), It.IsAny<IEnumerable<int>>(), requestContext)).Returns(new Message() { Value = new byte[] { 0, 1, 2, 3 } }); var twoQueuesStreamMapper = new HashRingBasedStreamQueueMapper(2, _options.TopicName); var twoQueuesOptions = new KafkaStreamProviderOptions(_options.ConnectionStrings, _options.TopicName, _options.ConsumerGroupName){NumOfQueues = 2}; KafkaQueueAdapter adapter = new KafkaQueueAdapter(_streamQueueMapper, _options, _providerName, factoryMock.Object, _logger); var first = Guid.NewGuid(); var second = Guid.NewGuid(); // Becuase we cannot mock the queue mapper.. we need to make sure we two guids that will return different queues... bool willGiveDifferentQueue = !(twoQueuesStreamMapper.GetQueueForStream(first, "test").Equals(twoQueuesStreamMapper.GetQueueForStream(second, "otherTest"))); while (!willGiveDifferentQueue) { second = Guid.NewGuid(); willGiveDifferentQueue = !(twoQueuesStreamMapper.GetQueueForStream(first, "test").Equals(twoQueuesStreamMapper.GetQueueForStream(second, "otherTest"))); } Task.WaitAll(adapter.QueueMessageBatchAsync(first, "test", new List<int>() { 1, 2, 3, 4 }, null, requestContext), adapter.QueueMessageBatchAsync(second, "otherTest", new List<int>() { 1, 2, 3, 4 }, null, requestContext)); }
public async Task QueueMessageBatchAsyncAllNoAck() { Mock<IKafkaBatchFactory> factoryMock = new Mock<IKafkaBatchFactory>(); Dictionary<string, object> requestContext = new Dictionary<string, object>(); factoryMock.Setup(x => x.ToKafkaMessage(It.IsAny<Guid>(), It.IsAny<string>(), It.IsAny<IEnumerable<int>>(), requestContext)).Returns((Message)null); KafkaStreamProviderOptions differentOptions = new KafkaStreamProviderOptions(_options.ConnectionStrings, _options.TopicName, _options.ConsumerGroupName) {AckLevel = 0}; KafkaQueueAdapter adapter = new KafkaQueueAdapter(_streamQueueMapper, differentOptions, _providerName, factoryMock.Object, _logger); Guid myGuid = Guid.NewGuid(); await adapter.QueueMessageBatchAsync(myGuid, "Test", new List<int>() { 1, 2, 3, 4 }, null, requestContext); }