public async Task InitializeConsumerGroupDoesntExistRecoveryTest()
{
var wantedOffset = 10;
var savedNewOffset = false;
Mock <IManualConsumer> consumerMock = new Mock <IManualConsumer>();
Mock <IKafkaBatchFactory> factoryMock = new Mock <IKafkaBatchFactory>();
KafkaStreamProviderOptions mockOptions = new KafkaStreamProviderOptions(_options.ConnectionStrings,
_options.TopicName, _options.ConsumerGroupName)
{
ShouldInitWithLastOffset = false
};
consumerMock.Setup(x => x.FetchOffset(_options.ConsumerGroupName)).Throws(new KafkaApplicationException("Test")
{
ErrorCode = (int)ErrorResponseCode.UnknownTopicOrPartition
});
KafkaQueueAdapterReceiver adapterReceiver = new KafkaQueueAdapterReceiver(_id, consumerMock.Object, _options, factoryMock.Object, _logger);
consumerMock.Setup(x => x.FetchLastOffset()).ReturnsAsync(wantedOffset).Callback(() => savedNewOffset = true);
await adapterReceiver.Initialize(TimeSpan.MaxValue);
if (savedNewOffset)
{
consumerMock.Setup(x => x.FetchOffset(_options.ConsumerGroupName)).ReturnsAsync(wantedOffset);
}
await adapterReceiver.Initialize(TimeSpan.MaxValue);
Assert.AreEqual(wantedOffset, adapterReceiver.CurrentOffset);
}
public KafkaQueueAdapterUnitTests()
{
Mock <Logger> loggerMock = new Mock <Logger>();
var connectionStrings = new List <Uri> {
new Uri("http://kafka1:9092;http://kafka2:9092")
};
var topicName = "Jonathan.ab.KafkaStreamProviderTestsNew";
var consumerGroupName = "TestConsumerGroupName";
_logger = loggerMock.Object;
_options = new KafkaStreamProviderOptions(connectionStrings.ToArray(), topicName, consumerGroupName);
_streamQueueMapper = new HashRingBasedStreamQueueMapper(_options.NumOfQueues, _options.TopicName);
}
public KafkaQueueAdapterReceiverUnitTests()
{
Mock <Logger> loggerMock = new Mock <Logger>();
_logger = loggerMock.Object;
var connectionStrings = new List <Uri> {
new Uri("http://192.168.10.27:9092")
};
var topicName = "TestTopic";
var consumerGroupName = "TestConsumerGroup";
_options = new KafkaStreamProviderOptions(connectionStrings.ToArray(), topicName, consumerGroupName);
_id = QueueId.GetQueueId("test", 0, 0);
}
public async Task QueueMessageBatchAsyncAllNoAck()
{
Mock <IKafkaBatchFactory> factoryMock = new Mock <IKafkaBatchFactory>();
Dictionary <string, object> requestContext = new Dictionary <string, object>();
factoryMock.Setup(x => x.ToKafkaMessage(It.IsAny <Guid>(), It.IsAny <string>(), It.IsAny <IEnumerable <int> >(), requestContext)).Returns((Message)null);
KafkaStreamProviderOptions differentOptions = new KafkaStreamProviderOptions(_options.ConnectionStrings,
_options.TopicName, _options.ConsumerGroupName)
{
AckLevel = 0
};
KafkaQueueAdapter adapter = new KafkaQueueAdapter(_streamQueueMapper, differentOptions, _providerName, factoryMock.Object, _logger);
Guid myGuid = Guid.NewGuid();
await adapter.QueueMessageBatchAsync(myGuid, "Test", new List <int>() { 1, 2, 3, 4 }, null, requestContext);
}
public async Task InitializeGetOffsetFromConsumerGroupTest()
{
var wantedOffset = 10;
Mock <IManualConsumer> consumerMock = new Mock <IManualConsumer>();
Mock <IKafkaBatchFactory> factoryMock = new Mock <IKafkaBatchFactory>();
KafkaStreamProviderOptions mockOptions = new KafkaStreamProviderOptions(_options.ConnectionStrings,
_options.TopicName, _options.ConsumerGroupName)
{
ShouldInitWithLastOffset = false
};
consumerMock.Setup(x => x.FetchOffset(_options.ConsumerGroupName)).ReturnsAsync(wantedOffset);
KafkaQueueAdapterReceiver adapterReceiver = new KafkaQueueAdapterReceiver(_id, consumerMock.Object, mockOptions, factoryMock.Object, _logger);
await adapterReceiver.Initialize(TimeSpan.MaxValue);
Assert.AreEqual(wantedOffset, adapterReceiver.CurrentOffset);
}
public void QueueMessageBatchAsyncQueueingTwiceDifferentQueuesTest()
{
Mock <IKafkaBatchFactory> factoryMock = new Mock <IKafkaBatchFactory>();
Dictionary <string, object> requestContext = new Dictionary <string, object>();
factoryMock.Setup(x => x.ToKafkaMessage(It.IsAny <Guid>(), It.IsAny <string>(), It.IsAny <IEnumerable <int> >(), requestContext)).Returns(new Message()
{
Value = new byte[] { 0, 1, 2, 3 }
});
var twoQueuesStreamMapper = new HashRingBasedStreamQueueMapper(2, _options.TopicName);
var twoQueuesOptions = new KafkaStreamProviderOptions(_options.ConnectionStrings, _options.TopicName, _options.ConsumerGroupName)
{
NumOfQueues = 2
};
KafkaQueueAdapter adapter = new KafkaQueueAdapter(_streamQueueMapper, _options, _providerName, factoryMock.Object, _logger);
var first = Guid.NewGuid();
var second = Guid.NewGuid();
// Becuase we cannot mock the queue mapper.. we need to make sure we two guids that will return different queues...
bool willGiveDifferentQueue = !(twoQueuesStreamMapper.GetQueueForStream(first, "test").Equals(twoQueuesStreamMapper.GetQueueForStream(second, "otherTest")));
while (!willGiveDifferentQueue)
{
second = Guid.NewGuid();
willGiveDifferentQueue = !(twoQueuesStreamMapper.GetQueueForStream(first, "test").Equals(twoQueuesStreamMapper.GetQueueForStream(second, "otherTest")));
}
Task.WaitAll(adapter.QueueMessageBatchAsync(first, "test", new List <int>()
{
1, 2, 3, 4
}, null, requestContext),
adapter.QueueMessageBatchAsync(second, "otherTest", new List <int>()
{
1, 2, 3, 4
}, null, requestContext));
}
public async Task GetMessagesQueueSavingBatchAfterFewFetchesTest()
{
bool hasCommited = false;
Message message1 = new Message()
{
Value = new byte[] { 1, 2, 3, 4 }, Meta = new MessageMetadata()
{
Offset = 0, PartitionId = 0
}
};
Message message2 = new Message()
{
Value = new byte[] { 4, 5, 6, 7 }, Meta = new MessageMetadata()
{
Offset = 1, PartitionId = 0
}
};
Message message3 = new Message()
{
Value = new byte[] { 8, 9, 10, 11 }, Meta = new MessageMetadata()
{
Offset = 2, PartitionId = 0
}
};
Message message4 = new Message()
{
Value = new byte[] { 12, 13, 14, 15 }, Meta = new MessageMetadata()
{
Offset = 3, PartitionId = 0
}
};
Message message5 = new Message()
{
Value = new byte[] { 16, 17, 18, 19 }, Meta = new MessageMetadata()
{
Offset = 4, PartitionId = 0
}
};
Message message6 = new Message()
{
Value = new byte[] { 21, 22, 23, 24 }, Meta = new MessageMetadata()
{
Offset = 5, PartitionId = 0
}
};
Mock <IManualConsumer> consumerMock = new Mock <IManualConsumer>();
Mock <IBatchContainer> batchContainerMock1 = new Mock <IBatchContainer>();
Mock <IBatchContainer> batchContainerMock2 = new Mock <IBatchContainer>();
Mock <IBatchContainer> batchContainerMock3 = new Mock <IBatchContainer>();
Mock <IBatchContainer> batchContainerMock4 = new Mock <IBatchContainer>();
Mock <IBatchContainer> batchContainerMock5 = new Mock <IBatchContainer>();
Mock <IBatchContainer> batchContainerMock6 = new Mock <IBatchContainer>();
var returnCollection1 = Task.FromResult <IEnumerable <Message> >(new List <Message>()
{
message1
});
consumerMock.Setup(x => x.FetchMessages(6, 0)).Returns(returnCollection1);
var returnCollection2 = Task.FromResult <IEnumerable <Message> >(new List <Message>()
{
message2, message3
});
consumerMock.Setup(x => x.FetchMessages(6, 1)).Returns(returnCollection2);
var returnCollection3 = Task.FromResult <IEnumerable <Message> >(new List <Message>()
{
message4, message5, message6
});
consumerMock.Setup(x => x.FetchMessages(6, 3)).Returns(returnCollection3);
consumerMock.Setup(x => x.FetchOffset(_options.ConsumerGroupName)).Returns(Task.FromResult <long>(0));
consumerMock.Setup(x => x.UpdateOrCreateOffset(_options.ConsumerGroupName, 3)).Returns(TaskDone.Done)
.Callback(() => hasCommited = true);
Mock <IKafkaBatchFactory> factoryMock = new Mock <IKafkaBatchFactory>();
factoryMock.Setup(x => x.FromKafkaMessage(message1, It.IsAny <long>()))
.Returns(batchContainerMock1.Object);
factoryMock.Setup(x => x.FromKafkaMessage(message2, It.IsAny <long>()))
.Returns(batchContainerMock2.Object);
factoryMock.Setup(x => x.FromKafkaMessage(message3, It.IsAny <long>()))
.Returns(batchContainerMock3.Object);
factoryMock.Setup(x => x.FromKafkaMessage(message4, It.IsAny <long>()))
.Returns(batchContainerMock4.Object);
factoryMock.Setup(x => x.FromKafkaMessage(message5, It.IsAny <long>()))
.Returns(batchContainerMock5.Object);
factoryMock.Setup(x => x.FromKafkaMessage(message6, It.IsAny <long>()))
.Returns(batchContainerMock6.Object);
var newOptions = new KafkaStreamProviderOptions(_options.ConnectionStrings, _options.TopicName, _options.ConsumerGroupName)
{
OffsetCommitInterval = 2, ShouldInitWithLastOffset = false
};
KafkaQueueAdapterReceiver adapterReceiver = new KafkaQueueAdapterReceiver(_id, consumerMock.Object, newOptions, factoryMock.Object, _logger);
await adapterReceiver.Initialize(TimeSpan.MaxValue);
int numToTake = 6;
// Asking once, should return 1 message
var result = await adapterReceiver.GetQueueMessagesAsync(6);
Assert.AreEqual(1, result.Count);
Assert.AreEqual(batchContainerMock1.Object, result[0]);
// Now sth has happened, restating the adapter, the offset should not be saved
adapterReceiver = new KafkaQueueAdapterReceiver(_id, consumerMock.Object, newOptions, factoryMock.Object, _logger);
await adapterReceiver.Initialize(TimeSpan.MaxValue);
// Asking once again, should return 1 message
result = await adapterReceiver.GetQueueMessagesAsync(6);
Assert.AreEqual(1, result.Count);
Assert.AreEqual(batchContainerMock1.Object, result[0]);
// Asking second again, should return 2 messages and commit
result = await adapterReceiver.GetQueueMessagesAsync(6);
Assert.AreEqual(2, result.Count);
Assert.AreEqual(batchContainerMock2.Object, result[0]);
Assert.AreEqual(batchContainerMock3.Object, result[1]);
if (hasCommited)
{
consumerMock.Setup(x => x.FetchOffset(_options.ConsumerGroupName)).Returns(Task.FromResult <long>(3));
}
// Now sth happened, but this time the offset was committed
adapterReceiver = new KafkaQueueAdapterReceiver(_id, consumerMock.Object, newOptions, factoryMock.Object, _logger);
await adapterReceiver.Initialize(TimeSpan.MaxValue);
// This request should return the last 3 messages
result = await adapterReceiver.GetQueueMessagesAsync(numToTake);
Assert.AreEqual(3, result.Count);
Assert.AreEqual(batchContainerMock4.Object, result[0]);
Assert.AreEqual(batchContainerMock5.Object, result[1]);
Assert.AreEqual(batchContainerMock6.Object, result[2]);
}
