public void Init(IProviderConfiguration config, string providerName, Logger logger, IServiceProvider serviceProvider)
{
if (config == null)
{
throw new ArgumentNullException(nameof(config));
}
if (logger == null)
{
throw new ArgumentNullException(nameof(logger));
}
if (String.IsNullOrEmpty(providerName))
{
throw new ArgumentNullException(nameof(providerName));
}
// Creating an options object with all the config values
_options = new KafkaStreamProviderOptions(config);
if (!_options.UsingExternalMetrics)
{
Metric.Config.WithHttpEndpoint($"http://localhost:{_options.MetricsPort}/");
}
if (!_options.IncludeMetrics)
{
Metric.Context("KafkaStreamProvider").Advanced.CompletelyDisableMetrics();
}
_providerName = providerName;
_streamQueueMapper = new HashRingBasedStreamQueueMapper(_options.NumOfQueues, providerName);
_logger = logger;
_adapter = new KafkaQueueAdapter(_streamQueueMapper, _options, providerName, new KafkaBatchFactory(), _logger);
_adapterCache = new TimedQueueAdapterCache(this, TimeSpan.FromSeconds(_options.CacheTimespanInSeconds), _options.CacheSize, _options.CacheNumOfBuckets, logger);
}
public KafkaQueueAdapterReceiver(QueueId queueId, IManualConsumer consumer, KafkaStreamProviderOptions options,
IKafkaBatchFactory factory, ILogger logger)
{
// input checks
if (queueId == null)
{
throw new ArgumentNullException(nameof(queueId));
}
if (consumer == null)
{
throw new ArgumentNullException(nameof(consumer));
}
if (factory == null)
{
throw new ArgumentNullException(nameof(factory));
}
if (options == null)
{
throw new ArgumentNullException(nameof(options));
}
if (logger == null)
{
throw new ArgumentNullException(nameof(logger));
}
_counterCurrentOffset = Metrics.Metric.Context("KafkaStreamProvider").Counter($"CurrentOffset queueId:({queueId.GetNumericId()})", unit: Unit.Custom("Log"));
_options = options;
Id = queueId;
_consumer = consumer;
_factory = factory;
_logger = logger;
}
public void Init(IProviderConfiguration config, string providerName, Logger logger, IServiceProvider serviceProvider)
{
if (config == null) throw new ArgumentNullException(nameof(config));
if (logger == null) throw new ArgumentNullException(nameof(logger));
if (String.IsNullOrEmpty(providerName)) throw new ArgumentNullException(nameof(providerName));
// Creating an options object with all the config values
_options = new KafkaStreamProviderOptions(config);
if (!_options.UsingExternalMetrics)
{
Metric.Config.WithHttpEndpoint($"http://localhost:{_options.MetricsPort}/");
}
if (!_options.IncludeMetrics)
{
Metric.Context("KafkaStreamProvider").Advanced.CompletelyDisableMetrics();
}
_providerName = providerName;
_streamQueueMapper = new HashRingBasedStreamQueueMapper(_options.NumOfQueues, providerName);
_logger = logger;
_adapter = new KafkaQueueAdapter(_streamQueueMapper, _options, providerName, new KafkaBatchFactory(), _logger);
_adapterCache = new TimedQueueAdapterCache(this, TimeSpan.FromSeconds(_options.CacheTimespanInSeconds), _options.CacheSize, _options.CacheNumOfBuckets, logger);
}
public KafkaQueueAdapterReceiverUnitTests()
{
Mock<Logger> loggerMock = new Mock<Logger>();
_logger = loggerMock.Object;
var connectionStrings = new List<Uri> {new Uri("http://192.168.10.27:9092")};
var topicName = "TestTopic";
var consumerGroupName = "TestConsumerGroup";
_options = new KafkaStreamProviderOptions(connectionStrings.ToArray(), topicName, consumerGroupName);
_id = QueueId.GetQueueId("test", 0, 0);
}
public KafkaQueueAdapterUnitTests()
{
Mock<Logger> loggerMock = new Mock<Logger>();
var connectionStrings = new List<Uri> {new Uri("http://kafka1:9092;http://kafka2:9092") };
var topicName = "Jonathan.ab.KafkaStreamProviderTestsNew";
var consumerGroupName = "TestConsumerGroupName";
_logger = loggerMock.Object;
_options = new KafkaStreamProviderOptions(connectionStrings.ToArray(), topicName, consumerGroupName);
_streamQueueMapper = new HashRingBasedStreamQueueMapper(_options.NumOfQueues, _options.TopicName);
}
public KafkaQueueAdapter(HashRingBasedStreamQueueMapper queueMapper, KafkaStreamProviderOptions options,
string providerName, IKafkaBatchFactory batchFactory, ILogger logger)
{
if (options == null)
{
throw new ArgumentNullException(nameof(options));
}
if (batchFactory == null)
{
throw new ArgumentNullException(nameof(batchFactory));
}
if (queueMapper == null)
{
throw new ArgumentNullException(nameof(queueMapper));
}
if (logger == null)
{
throw new ArgumentNullException(nameof(logger));
}
if (string.IsNullOrEmpty(providerName))
{
throw new ArgumentNullException(nameof(providerName));
}
_options = options;
_streamQueueMapper = queueMapper;
Name = providerName;
_batchFactory = batchFactory;
_logger = logger;
// Creating a producer
KafkaOptions kafkaOptions = new KafkaOptions(_options.ConnectionStrings.ToArray())
{
Log = new KafkaLogBridge(logger)
};
var broker = new BrokerRouter(kafkaOptions);
_producer = new Producer(broker)
{
BatchDelayTime = TimeSpan.FromMilliseconds(_options.TimeToWaitForBatchInMs),
BatchSize = _options.ProduceBatchSize
};
_gateway = new ProtocolGateway(kafkaOptions);
_logger.Info("KafkaQueueAdapter - Created a KafkaQueueAdapter");
}
public KafkaQueueAdapterReceiver(QueueId queueId, IManualConsumer consumer, KafkaStreamProviderOptions options,
IKafkaBatchFactory factory, Logger logger)
{
// input checks
if (queueId == null) throw new ArgumentNullException(nameof(queueId));
if (consumer == null) throw new ArgumentNullException(nameof(consumer));
if (factory == null) throw new ArgumentNullException(nameof(factory));
if (options == null) throw new ArgumentNullException(nameof(options));
if (logger == null) throw new ArgumentNullException(nameof(logger));
_counterCurrentOffset = Metric.Context("KafkaStreamProvider").Counter($"CurrentOffset queueId:({queueId.GetNumericId()})", unit: Unit.Custom("Log"));
_options = options;
Id = queueId;
_consumer = consumer;
_factory = factory;
_logger = logger;
}
public async Task GetMessagesQueueSavingBatchAfterFewFetchesTest()
{
bool hasCommited = false;
Message message1 = new Message() { Value = new byte[] { 1, 2, 3, 4 }, Meta = new MessageMetadata() { Offset = 0, PartitionId = 0 } };
Message message2 = new Message() { Value = new byte[] { 4, 5, 6, 7 }, Meta = new MessageMetadata() { Offset = 1, PartitionId = 0 } };
Message message3 = new Message() { Value = new byte[] { 8, 9, 10, 11 }, Meta = new MessageMetadata() { Offset = 2, PartitionId = 0 } };
Message message4 = new Message() { Value = new byte[] { 12, 13, 14, 15 }, Meta = new MessageMetadata() { Offset = 3, PartitionId = 0 } };
Message message5 = new Message() { Value = new byte[] { 16, 17, 18, 19 }, Meta = new MessageMetadata() { Offset = 4, PartitionId = 0 } };
Message message6 = new Message() { Value = new byte[] { 21, 22, 23, 24 }, Meta = new MessageMetadata() { Offset = 5, PartitionId = 0 } };
Mock<IManualConsumer> consumerMock = new Mock<IManualConsumer>();
Mock<IBatchContainer> batchContainerMock1 = new Mock<IBatchContainer>();
Mock<IBatchContainer> batchContainerMock2 = new Mock<IBatchContainer>();
Mock<IBatchContainer> batchContainerMock3 = new Mock<IBatchContainer>();
Mock<IBatchContainer> batchContainerMock4 = new Mock<IBatchContainer>();
Mock<IBatchContainer> batchContainerMock5 = new Mock<IBatchContainer>();
Mock<IBatchContainer> batchContainerMock6 = new Mock<IBatchContainer>();
var returnCollection1 = Task.FromResult<IEnumerable<Message>>(new List<Message>() { message1 });
consumerMock.Setup(x => x.FetchMessages(6, 0)).Returns(returnCollection1);
var returnCollection2 = Task.FromResult<IEnumerable<Message>>(new List<Message>() { message2, message3});
consumerMock.Setup(x => x.FetchMessages(6, 1)).Returns(returnCollection2);
var returnCollection3 = Task.FromResult<IEnumerable<Message>>(new List<Message>() { message4, message5, message6 });
consumerMock.Setup(x => x.FetchMessages(6, 3)).Returns(returnCollection3);
consumerMock.Setup(x => x.FetchOffset(_options.ConsumerGroupName)).Returns(Task.FromResult<long>(0));
consumerMock.Setup(x => x.UpdateOrCreateOffset(_options.ConsumerGroupName, 3)).Returns(TaskDone.Done)
.Callback(() => hasCommited = true);
Mock<IKafkaBatchFactory> factoryMock = new Mock<IKafkaBatchFactory>();
factoryMock.Setup(x => x.FromKafkaMessage(message1, It.IsAny<long>()))
.Returns(batchContainerMock1.Object);
factoryMock.Setup(x => x.FromKafkaMessage(message2, It.IsAny<long>()))
.Returns(batchContainerMock2.Object);
factoryMock.Setup(x => x.FromKafkaMessage(message3, It.IsAny<long>()))
.Returns(batchContainerMock3.Object);
factoryMock.Setup(x => x.FromKafkaMessage(message4, It.IsAny<long>()))
.Returns(batchContainerMock4.Object);
factoryMock.Setup(x => x.FromKafkaMessage(message5, It.IsAny<long>()))
.Returns(batchContainerMock5.Object);
factoryMock.Setup(x => x.FromKafkaMessage(message6, It.IsAny<long>()))
.Returns(batchContainerMock6.Object);
var newOptions = new KafkaStreamProviderOptions(_options.ConnectionStrings, _options.TopicName, _options.ConsumerGroupName){OffsetCommitInterval = 2, ShouldInitWithLastOffset = false};
KafkaQueueAdapterReceiver adapterReceiver = new KafkaQueueAdapterReceiver(_id, consumerMock.Object, newOptions, factoryMock.Object, _logger);
await adapterReceiver.Initialize(TimeSpan.MaxValue);
int numToTake = 6;
// Asking once, should return 1 message
var result = await adapterReceiver.GetQueueMessagesAsync(6);
Assert.AreEqual(1, result.Count);
Assert.AreEqual(batchContainerMock1.Object, result[0]);
// Now sth has happened, restating the adapter, the offset should not be saved
adapterReceiver = new KafkaQueueAdapterReceiver(_id, consumerMock.Object, newOptions, factoryMock.Object, _logger);
await adapterReceiver.Initialize(TimeSpan.MaxValue);
// Asking once again, should return 1 message
result = await adapterReceiver.GetQueueMessagesAsync(6);
Assert.AreEqual(1, result.Count);
Assert.AreEqual(batchContainerMock1.Object, result[0]);
// Asking second again, should return 2 messages and commit
result = await adapterReceiver.GetQueueMessagesAsync(6);
Assert.AreEqual(2, result.Count);
Assert.AreEqual(batchContainerMock2.Object, result[0]);
Assert.AreEqual(batchContainerMock3.Object, result[1]);
if (hasCommited)
{
consumerMock.Setup(x => x.FetchOffset(_options.ConsumerGroupName)).Returns(Task.FromResult<long>(3));
}
// Now sth happened, but this time the offset was committed
adapterReceiver = new KafkaQueueAdapterReceiver(_id, consumerMock.Object, newOptions, factoryMock.Object, _logger);
await adapterReceiver.Initialize(TimeSpan.MaxValue);
// This request should return the last 3 messages
result = await adapterReceiver.GetQueueMessagesAsync(numToTake);
Assert.AreEqual(3, result.Count);
Assert.AreEqual(batchContainerMock4.Object, result[0]);
Assert.AreEqual(batchContainerMock5.Object, result[1]);
Assert.AreEqual(batchContainerMock6.Object, result[2]);
}
public async Task InitializeConsumerGroupDoesntExistRecoveryTest()
{
var wantedOffset = 10;
var savedNewOffset = false;
Mock<IManualConsumer> consumerMock = new Mock<IManualConsumer>();
Mock<IKafkaBatchFactory> factoryMock = new Mock<IKafkaBatchFactory>();
KafkaStreamProviderOptions mockOptions = new KafkaStreamProviderOptions(_options.ConnectionStrings,
_options.TopicName, _options.ConsumerGroupName) {ShouldInitWithLastOffset = false};
consumerMock.Setup(x => x.FetchOffset(_options.ConsumerGroupName)).Throws(new KafkaApplicationException("Test") { ErrorCode = (int)ErrorResponseCode.UnknownTopicOrPartition });
KafkaQueueAdapterReceiver adapterReceiver = new KafkaQueueAdapterReceiver(_id, consumerMock.Object, _options, factoryMock.Object, _logger);
consumerMock.Setup(x => x.FetchLastOffset()).ReturnsAsync(wantedOffset).Callback(() => savedNewOffset=true);
await adapterReceiver.Initialize(TimeSpan.MaxValue);
if (savedNewOffset)
{
consumerMock.Setup(x => x.FetchOffset(_options.ConsumerGroupName)).ReturnsAsync(wantedOffset);
}
await adapterReceiver.Initialize(TimeSpan.MaxValue);
Assert.AreEqual(wantedOffset, adapterReceiver.CurrentOffset);
}
public async Task InitializeGetOffsetFromConsumerGroupTest()
{
var wantedOffset = 10;
Mock<IManualConsumer> consumerMock = new Mock<IManualConsumer>();
Mock<IKafkaBatchFactory> factoryMock = new Mock<IKafkaBatchFactory>();
KafkaStreamProviderOptions mockOptions = new KafkaStreamProviderOptions(_options.ConnectionStrings,
_options.TopicName, _options.ConsumerGroupName) {ShouldInitWithLastOffset = false};
consumerMock.Setup(x => x.FetchOffset(_options.ConsumerGroupName)).ReturnsAsync(wantedOffset);
KafkaQueueAdapterReceiver adapterReceiver = new KafkaQueueAdapterReceiver(_id, consumerMock.Object, mockOptions, factoryMock.Object, _logger);
await adapterReceiver.Initialize(TimeSpan.MaxValue);
Assert.AreEqual(wantedOffset, adapterReceiver.CurrentOffset);
}
public void QueueMessageBatchAsyncQueueingTwiceDifferentQueuesTest()
{
Mock<IKafkaBatchFactory> factoryMock = new Mock<IKafkaBatchFactory>();
Dictionary<string, object> requestContext = new Dictionary<string, object>();
factoryMock.Setup(x => x.ToKafkaMessage(It.IsAny<Guid>(), It.IsAny<string>(), It.IsAny<IEnumerable<int>>(), requestContext)).Returns(new Message() { Value = new byte[] { 0, 1, 2, 3 } });
var twoQueuesStreamMapper = new HashRingBasedStreamQueueMapper(2, _options.TopicName);
var twoQueuesOptions = new KafkaStreamProviderOptions(_options.ConnectionStrings, _options.TopicName, _options.ConsumerGroupName){NumOfQueues = 2};
KafkaQueueAdapter adapter = new KafkaQueueAdapter(_streamQueueMapper, _options, _providerName, factoryMock.Object, _logger);
var first = Guid.NewGuid();
var second = Guid.NewGuid();
// Becuase we cannot mock the queue mapper.. we need to make sure we two guids that will return different queues...
bool willGiveDifferentQueue = !(twoQueuesStreamMapper.GetQueueForStream(first, "test").Equals(twoQueuesStreamMapper.GetQueueForStream(second, "otherTest")));
while (!willGiveDifferentQueue)
{
second = Guid.NewGuid();
willGiveDifferentQueue = !(twoQueuesStreamMapper.GetQueueForStream(first, "test").Equals(twoQueuesStreamMapper.GetQueueForStream(second, "otherTest")));
}
Task.WaitAll(adapter.QueueMessageBatchAsync(first, "test", new List<int>() { 1, 2, 3, 4 }, null, requestContext),
adapter.QueueMessageBatchAsync(second, "otherTest", new List<int>() { 1, 2, 3, 4 }, null, requestContext));
}
public async Task QueueMessageBatchAsyncAllNoAck()
{
Mock<IKafkaBatchFactory> factoryMock = new Mock<IKafkaBatchFactory>();
Dictionary<string, object> requestContext = new Dictionary<string, object>();
factoryMock.Setup(x => x.ToKafkaMessage(It.IsAny<Guid>(), It.IsAny<string>(), It.IsAny<IEnumerable<int>>(), requestContext)).Returns((Message)null);
KafkaStreamProviderOptions differentOptions = new KafkaStreamProviderOptions(_options.ConnectionStrings,
_options.TopicName, _options.ConsumerGroupName) {AckLevel = 0};
KafkaQueueAdapter adapter = new KafkaQueueAdapter(_streamQueueMapper, differentOptions, _providerName, factoryMock.Object, _logger);
Guid myGuid = Guid.NewGuid();
await adapter.QueueMessageBatchAsync(myGuid, "Test", new List<int>() { 1, 2, 3, 4 }, null, requestContext);
}