public async Task InitializeConsumerGroupDoesntExistRecoveryTest() { var wantedOffset = 10; var savedNewOffset = false; Mock <IManualConsumer> consumerMock = new Mock <IManualConsumer>(); Mock <IKafkaBatchFactory> factoryMock = new Mock <IKafkaBatchFactory>(); KafkaStreamProviderOptions mockOptions = new KafkaStreamProviderOptions(_options.ConnectionStrings, _options.TopicName, _options.ConsumerGroupName) { ShouldInitWithLastOffset = false }; consumerMock.Setup(x => x.FetchOffset(_options.ConsumerGroupName)).Throws(new KafkaApplicationException("Test") { ErrorCode = (int)ErrorResponseCode.UnknownTopicOrPartition }); KafkaQueueAdapterReceiver adapterReceiver = new KafkaQueueAdapterReceiver(_id, consumerMock.Object, _options, factoryMock.Object, _logger); consumerMock.Setup(x => x.FetchLastOffset()).ReturnsAsync(wantedOffset).Callback(() => savedNewOffset = true); await adapterReceiver.Initialize(TimeSpan.MaxValue); if (savedNewOffset) { consumerMock.Setup(x => x.FetchOffset(_options.ConsumerGroupName)).ReturnsAsync(wantedOffset); } await adapterReceiver.Initialize(TimeSpan.MaxValue); Assert.AreEqual(wantedOffset, adapterReceiver.CurrentOffset); }
public KafkaQueueAdapterUnitTests() { Mock <Logger> loggerMock = new Mock <Logger>(); var connectionStrings = new List <Uri> { new Uri("http://kafka1:9092;http://kafka2:9092") }; var topicName = "Jonathan.ab.KafkaStreamProviderTestsNew"; var consumerGroupName = "TestConsumerGroupName"; _logger = loggerMock.Object; _options = new KafkaStreamProviderOptions(connectionStrings.ToArray(), topicName, consumerGroupName); _streamQueueMapper = new HashRingBasedStreamQueueMapper(_options.NumOfQueues, _options.TopicName); }
public KafkaQueueAdapterReceiverUnitTests() { Mock <Logger> loggerMock = new Mock <Logger>(); _logger = loggerMock.Object; var connectionStrings = new List <Uri> { new Uri("http://192.168.10.27:9092") }; var topicName = "TestTopic"; var consumerGroupName = "TestConsumerGroup"; _options = new KafkaStreamProviderOptions(connectionStrings.ToArray(), topicName, consumerGroupName); _id = QueueId.GetQueueId("test", 0, 0); }
public async Task QueueMessageBatchAsyncAllNoAck() { Mock <IKafkaBatchFactory> factoryMock = new Mock <IKafkaBatchFactory>(); Dictionary <string, object> requestContext = new Dictionary <string, object>(); factoryMock.Setup(x => x.ToKafkaMessage(It.IsAny <Guid>(), It.IsAny <string>(), It.IsAny <IEnumerable <int> >(), requestContext)).Returns((Message)null); KafkaStreamProviderOptions differentOptions = new KafkaStreamProviderOptions(_options.ConnectionStrings, _options.TopicName, _options.ConsumerGroupName) { AckLevel = 0 }; KafkaQueueAdapter adapter = new KafkaQueueAdapter(_streamQueueMapper, differentOptions, _providerName, factoryMock.Object, _logger); Guid myGuid = Guid.NewGuid(); await adapter.QueueMessageBatchAsync(myGuid, "Test", new List <int>() { 1, 2, 3, 4 }, null, requestContext); }
public async Task InitializeGetOffsetFromConsumerGroupTest() { var wantedOffset = 10; Mock <IManualConsumer> consumerMock = new Mock <IManualConsumer>(); Mock <IKafkaBatchFactory> factoryMock = new Mock <IKafkaBatchFactory>(); KafkaStreamProviderOptions mockOptions = new KafkaStreamProviderOptions(_options.ConnectionStrings, _options.TopicName, _options.ConsumerGroupName) { ShouldInitWithLastOffset = false }; consumerMock.Setup(x => x.FetchOffset(_options.ConsumerGroupName)).ReturnsAsync(wantedOffset); KafkaQueueAdapterReceiver adapterReceiver = new KafkaQueueAdapterReceiver(_id, consumerMock.Object, mockOptions, factoryMock.Object, _logger); await adapterReceiver.Initialize(TimeSpan.MaxValue); Assert.AreEqual(wantedOffset, adapterReceiver.CurrentOffset); }
public void QueueMessageBatchAsyncQueueingTwiceDifferentQueuesTest() { Mock <IKafkaBatchFactory> factoryMock = new Mock <IKafkaBatchFactory>(); Dictionary <string, object> requestContext = new Dictionary <string, object>(); factoryMock.Setup(x => x.ToKafkaMessage(It.IsAny <Guid>(), It.IsAny <string>(), It.IsAny <IEnumerable <int> >(), requestContext)).Returns(new Message() { Value = new byte[] { 0, 1, 2, 3 } }); var twoQueuesStreamMapper = new HashRingBasedStreamQueueMapper(2, _options.TopicName); var twoQueuesOptions = new KafkaStreamProviderOptions(_options.ConnectionStrings, _options.TopicName, _options.ConsumerGroupName) { NumOfQueues = 2 }; KafkaQueueAdapter adapter = new KafkaQueueAdapter(_streamQueueMapper, _options, _providerName, factoryMock.Object, _logger); var first = Guid.NewGuid(); var second = Guid.NewGuid(); // Becuase we cannot mock the queue mapper.. we need to make sure we two guids that will return different queues... bool willGiveDifferentQueue = !(twoQueuesStreamMapper.GetQueueForStream(first, "test").Equals(twoQueuesStreamMapper.GetQueueForStream(second, "otherTest"))); while (!willGiveDifferentQueue) { second = Guid.NewGuid(); willGiveDifferentQueue = !(twoQueuesStreamMapper.GetQueueForStream(first, "test").Equals(twoQueuesStreamMapper.GetQueueForStream(second, "otherTest"))); } Task.WaitAll(adapter.QueueMessageBatchAsync(first, "test", new List <int>() { 1, 2, 3, 4 }, null, requestContext), adapter.QueueMessageBatchAsync(second, "otherTest", new List <int>() { 1, 2, 3, 4 }, null, requestContext)); }
public async Task GetMessagesQueueSavingBatchAfterFewFetchesTest() { bool hasCommited = false; Message message1 = new Message() { Value = new byte[] { 1, 2, 3, 4 }, Meta = new MessageMetadata() { Offset = 0, PartitionId = 0 } }; Message message2 = new Message() { Value = new byte[] { 4, 5, 6, 7 }, Meta = new MessageMetadata() { Offset = 1, PartitionId = 0 } }; Message message3 = new Message() { Value = new byte[] { 8, 9, 10, 11 }, Meta = new MessageMetadata() { Offset = 2, PartitionId = 0 } }; Message message4 = new Message() { Value = new byte[] { 12, 13, 14, 15 }, Meta = new MessageMetadata() { Offset = 3, PartitionId = 0 } }; Message message5 = new Message() { Value = new byte[] { 16, 17, 18, 19 }, Meta = new MessageMetadata() { Offset = 4, PartitionId = 0 } }; Message message6 = new Message() { Value = new byte[] { 21, 22, 23, 24 }, Meta = new MessageMetadata() { Offset = 5, PartitionId = 0 } }; Mock <IManualConsumer> consumerMock = new Mock <IManualConsumer>(); Mock <IBatchContainer> batchContainerMock1 = new Mock <IBatchContainer>(); Mock <IBatchContainer> batchContainerMock2 = new Mock <IBatchContainer>(); Mock <IBatchContainer> batchContainerMock3 = new Mock <IBatchContainer>(); Mock <IBatchContainer> batchContainerMock4 = new Mock <IBatchContainer>(); Mock <IBatchContainer> batchContainerMock5 = new Mock <IBatchContainer>(); Mock <IBatchContainer> batchContainerMock6 = new Mock <IBatchContainer>(); var returnCollection1 = Task.FromResult <IEnumerable <Message> >(new List <Message>() { message1 }); consumerMock.Setup(x => x.FetchMessages(6, 0)).Returns(returnCollection1); var returnCollection2 = Task.FromResult <IEnumerable <Message> >(new List <Message>() { message2, message3 }); consumerMock.Setup(x => x.FetchMessages(6, 1)).Returns(returnCollection2); var returnCollection3 = Task.FromResult <IEnumerable <Message> >(new List <Message>() { message4, message5, message6 }); consumerMock.Setup(x => x.FetchMessages(6, 3)).Returns(returnCollection3); consumerMock.Setup(x => x.FetchOffset(_options.ConsumerGroupName)).Returns(Task.FromResult <long>(0)); consumerMock.Setup(x => x.UpdateOrCreateOffset(_options.ConsumerGroupName, 3)).Returns(TaskDone.Done) .Callback(() => hasCommited = true); Mock <IKafkaBatchFactory> factoryMock = new Mock <IKafkaBatchFactory>(); factoryMock.Setup(x => x.FromKafkaMessage(message1, It.IsAny <long>())) .Returns(batchContainerMock1.Object); factoryMock.Setup(x => x.FromKafkaMessage(message2, It.IsAny <long>())) .Returns(batchContainerMock2.Object); factoryMock.Setup(x => x.FromKafkaMessage(message3, It.IsAny <long>())) .Returns(batchContainerMock3.Object); factoryMock.Setup(x => x.FromKafkaMessage(message4, It.IsAny <long>())) .Returns(batchContainerMock4.Object); factoryMock.Setup(x => x.FromKafkaMessage(message5, It.IsAny <long>())) .Returns(batchContainerMock5.Object); factoryMock.Setup(x => x.FromKafkaMessage(message6, It.IsAny <long>())) .Returns(batchContainerMock6.Object); var newOptions = new KafkaStreamProviderOptions(_options.ConnectionStrings, _options.TopicName, _options.ConsumerGroupName) { OffsetCommitInterval = 2, ShouldInitWithLastOffset = false }; KafkaQueueAdapterReceiver adapterReceiver = new KafkaQueueAdapterReceiver(_id, consumerMock.Object, newOptions, factoryMock.Object, _logger); await adapterReceiver.Initialize(TimeSpan.MaxValue); int numToTake = 6; // Asking once, should return 1 message var result = await adapterReceiver.GetQueueMessagesAsync(6); Assert.AreEqual(1, result.Count); Assert.AreEqual(batchContainerMock1.Object, result[0]); // Now sth has happened, restating the adapter, the offset should not be saved adapterReceiver = new KafkaQueueAdapterReceiver(_id, consumerMock.Object, newOptions, factoryMock.Object, _logger); await adapterReceiver.Initialize(TimeSpan.MaxValue); // Asking once again, should return 1 message result = await adapterReceiver.GetQueueMessagesAsync(6); Assert.AreEqual(1, result.Count); Assert.AreEqual(batchContainerMock1.Object, result[0]); // Asking second again, should return 2 messages and commit result = await adapterReceiver.GetQueueMessagesAsync(6); Assert.AreEqual(2, result.Count); Assert.AreEqual(batchContainerMock2.Object, result[0]); Assert.AreEqual(batchContainerMock3.Object, result[1]); if (hasCommited) { consumerMock.Setup(x => x.FetchOffset(_options.ConsumerGroupName)).Returns(Task.FromResult <long>(3)); } // Now sth happened, but this time the offset was committed adapterReceiver = new KafkaQueueAdapterReceiver(_id, consumerMock.Object, newOptions, factoryMock.Object, _logger); await adapterReceiver.Initialize(TimeSpan.MaxValue); // This request should return the last 3 messages result = await adapterReceiver.GetQueueMessagesAsync(numToTake); Assert.AreEqual(3, result.Count); Assert.AreEqual(batchContainerMock4.Object, result[0]); Assert.AreEqual(batchContainerMock5.Object, result[1]); Assert.AreEqual(batchContainerMock6.Object, result[2]); }