public void Init(IProviderConfiguration config, string providerName, Logger logger, IServiceProvider serviceProvider)
{
if (config == null) throw new ArgumentNullException(nameof(config));
if (logger == null) throw new ArgumentNullException(nameof(logger));
if (String.IsNullOrEmpty(providerName)) throw new ArgumentNullException(nameof(providerName));
// Creating an options object with all the config values
_options = new KafkaStreamProviderOptions(config);
if (!_options.UsingExternalMetrics)
{
Metric.Config.WithHttpEndpoint($"http://localhost:{_options.MetricsPort}/");
}
if (!_options.IncludeMetrics)
{
Metric.Context("KafkaStreamProvider").Advanced.CompletelyDisableMetrics();
}
_providerName = providerName;
_streamQueueMapper = new HashRingBasedStreamQueueMapper(_options.NumOfQueues, providerName);
_logger = logger;
_adapter = new KafkaQueueAdapter(_streamQueueMapper, _options, providerName, new KafkaBatchFactory(), _logger);
_adapterCache = new TimedQueueAdapterCache(this, TimeSpan.FromSeconds(_options.CacheTimespanInSeconds), _options.CacheSize, _options.CacheNumOfBuckets, logger);
}
public Task<IQueueAdapter> CreateAdapter()
{
if (_adapter == null)
{
_adapter = new KafkaQueueAdapter(_streamQueueMapper, _options, _providerName,
new KafkaBatchFactory(), _logger);
}
return Task.FromResult<IQueueAdapter>(_adapter);
}
public async Task QueueMessageBatchAsyncAllMessagesAreFaultyTest()
{
Mock <IKafkaBatchFactory> factoryMock = new Mock <IKafkaBatchFactory>();
Dictionary <string, object> requestContext = new Dictionary <string, object>();
factoryMock.Setup(x => x.ToKafkaMessage(It.IsAny <Guid>(), It.IsAny <string>(), It.IsAny <IEnumerable <int> >(), requestContext)).Returns((Message)null);
KafkaQueueAdapter adapter = new KafkaQueueAdapter(_streamQueueMapper, _options, _providerName, factoryMock.Object, _logger);
Guid myGuid = Guid.NewGuid();
await adapter.QueueMessageBatchAsync(myGuid, "Test", new List <int>() { 1, 2, 3, 4 }, null, requestContext);
}
public async Task QueueMessageBatchAsyncAllNoAck()
{
Mock <IKafkaBatchFactory> factoryMock = new Mock <IKafkaBatchFactory>();
Dictionary <string, object> requestContext = new Dictionary <string, object>();
factoryMock.Setup(x => x.ToKafkaMessage(It.IsAny <Guid>(), It.IsAny <string>(), It.IsAny <IEnumerable <int> >(), requestContext)).Returns((Message)null);
KafkaStreamProviderOptions differentOptions = new KafkaStreamProviderOptions(_options.ConnectionStrings,
_options.TopicName, _options.ConsumerGroupName)
{
AckLevel = 0
};
KafkaQueueAdapter adapter = new KafkaQueueAdapter(_streamQueueMapper, differentOptions, _providerName, factoryMock.Object, _logger);
Guid myGuid = Guid.NewGuid();
await adapter.QueueMessageBatchAsync(myGuid, "Test", new List <int>() { 1, 2, 3, 4 }, null, requestContext);
}
public void QueueMessageBatchAsyncQueueingTwiceDifferentQueuesTest()
{
Mock <IKafkaBatchFactory> factoryMock = new Mock <IKafkaBatchFactory>();
Dictionary <string, object> requestContext = new Dictionary <string, object>();
factoryMock.Setup(x => x.ToKafkaMessage(It.IsAny <Guid>(), It.IsAny <string>(), It.IsAny <IEnumerable <int> >(), requestContext)).Returns(new Message()
{
Value = new byte[] { 0, 1, 2, 3 }
});
var twoQueuesStreamMapper = new HashRingBasedStreamQueueMapper(2, _options.TopicName);
var twoQueuesOptions = new KafkaStreamProviderOptions(_options.ConnectionStrings, _options.TopicName, _options.ConsumerGroupName)
{
NumOfQueues = 2
};
KafkaQueueAdapter adapter = new KafkaQueueAdapter(_streamQueueMapper, _options, _providerName, factoryMock.Object, _logger);
var first = Guid.NewGuid();
var second = Guid.NewGuid();
// Becuase we cannot mock the queue mapper.. we need to make sure we two guids that will return different queues...
bool willGiveDifferentQueue = !(twoQueuesStreamMapper.GetQueueForStream(first, "test").Equals(twoQueuesStreamMapper.GetQueueForStream(second, "otherTest")));
while (!willGiveDifferentQueue)
{
second = Guid.NewGuid();
willGiveDifferentQueue = !(twoQueuesStreamMapper.GetQueueForStream(first, "test").Equals(twoQueuesStreamMapper.GetQueueForStream(second, "otherTest")));
}
Task.WaitAll(adapter.QueueMessageBatchAsync(first, "test", new List <int>()
{
1, 2, 3, 4
}, null, requestContext),
adapter.QueueMessageBatchAsync(second, "otherTest", new List <int>()
{
1, 2, 3, 4
}, null, requestContext));
}
public void CtorTopicNullLoggerTest()
{
Mock <IKafkaBatchFactory> factoryMock = new Mock <IKafkaBatchFactory>();
var adapter = new KafkaQueueAdapter(_streamQueueMapper, _options, _providerName, factoryMock.Object, null);
}
public void CtorNullBatchFactoryTest()
{
var adapter = new KafkaQueueAdapter(_streamQueueMapper, _options, _providerName, null, _logger);
}
public void CtorNullQueueMapperTest()
{
Mock <IKafkaBatchFactory> factoryMock = new Mock <IKafkaBatchFactory>();
var adapter = new KafkaQueueAdapter(null, _options, _providerName, factoryMock.Object, _logger);
}
public void CtorEmptyProviderNameTest()
{
Mock <IKafkaBatchFactory> factoryMock = new Mock <IKafkaBatchFactory>();
var adapter = new KafkaQueueAdapter(_streamQueueMapper, _options, string.Empty, factoryMock.Object, _logger);
}
public void CtorTopicNullLoggerTest()
{
Mock<IKafkaBatchFactory> factoryMock = new Mock<IKafkaBatchFactory>();
var adapter = new KafkaQueueAdapter(_streamQueueMapper, _options, _providerName, factoryMock.Object, null);
}
public async Task QueueMessageBatchAsyncQueueingTwiceSameQueueTest()
{
Mock<IKafkaBatchFactory> factoryMock = new Mock<IKafkaBatchFactory>();
Dictionary<string, object> requestContext = new Dictionary<string, object>();
factoryMock.Setup(x => x.ToKafkaMessage(It.IsAny<Guid>(), It.IsAny<string>(), It.IsAny<IEnumerable<int>>(), requestContext)).Returns(new Message() { Value = new byte[] { 0, 1, 2, 3 } });
KafkaQueueAdapter adapter = new KafkaQueueAdapter(_streamQueueMapper, _options, _providerName, factoryMock.Object, _logger);
Guid myGuid = Guid.NewGuid();
await adapter.QueueMessageBatchAsync(myGuid, "Test", new List<int>() { 1, 2, 3, 4 }, null, requestContext);
await adapter.QueueMessageBatchAsync(myGuid, "Test", new List<int>() { 1, 2, 3, 4 }, null, requestContext);
}
public void CtorNullBatchFactoryTest()
{
var adapter = new KafkaQueueAdapter(_streamQueueMapper, _options, _providerName, null, _logger);
}
public void CtorNullQueueMapperTest()
{
Mock<IKafkaBatchFactory> factoryMock = new Mock<IKafkaBatchFactory>();
var adapter = new KafkaQueueAdapter(null, _options, _providerName, factoryMock.Object, _logger);
}
public void CtorEmptyProviderNameTest()
{
Mock<IKafkaBatchFactory> factoryMock = new Mock<IKafkaBatchFactory>();
var adapter = new KafkaQueueAdapter(_streamQueueMapper, _options, string.Empty, factoryMock.Object, _logger);
}
public void QueueMessageBatchAsyncQueueingTwiceDifferentQueuesTest()
{
Mock<IKafkaBatchFactory> factoryMock = new Mock<IKafkaBatchFactory>();
Dictionary<string, object> requestContext = new Dictionary<string, object>();
factoryMock.Setup(x => x.ToKafkaMessage(It.IsAny<Guid>(), It.IsAny<string>(), It.IsAny<IEnumerable<int>>(), requestContext)).Returns(new Message() { Value = new byte[] { 0, 1, 2, 3 } });
var twoQueuesStreamMapper = new HashRingBasedStreamQueueMapper(2, _options.TopicName);
var twoQueuesOptions = new KafkaStreamProviderOptions(_options.ConnectionStrings, _options.TopicName, _options.ConsumerGroupName){NumOfQueues = 2};
KafkaQueueAdapter adapter = new KafkaQueueAdapter(_streamQueueMapper, _options, _providerName, factoryMock.Object, _logger);
var first = Guid.NewGuid();
var second = Guid.NewGuid();
// Becuase we cannot mock the queue mapper.. we need to make sure we two guids that will return different queues...
bool willGiveDifferentQueue = !(twoQueuesStreamMapper.GetQueueForStream(first, "test").Equals(twoQueuesStreamMapper.GetQueueForStream(second, "otherTest")));
while (!willGiveDifferentQueue)
{
second = Guid.NewGuid();
willGiveDifferentQueue = !(twoQueuesStreamMapper.GetQueueForStream(first, "test").Equals(twoQueuesStreamMapper.GetQueueForStream(second, "otherTest")));
}
Task.WaitAll(adapter.QueueMessageBatchAsync(first, "test", new List<int>() { 1, 2, 3, 4 }, null, requestContext),
adapter.QueueMessageBatchAsync(second, "otherTest", new List<int>() { 1, 2, 3, 4 }, null, requestContext));
}
public async Task QueueMessageBatchAsyncAllNoAck()
{
Mock<IKafkaBatchFactory> factoryMock = new Mock<IKafkaBatchFactory>();
Dictionary<string, object> requestContext = new Dictionary<string, object>();
factoryMock.Setup(x => x.ToKafkaMessage(It.IsAny<Guid>(), It.IsAny<string>(), It.IsAny<IEnumerable<int>>(), requestContext)).Returns((Message)null);
KafkaStreamProviderOptions differentOptions = new KafkaStreamProviderOptions(_options.ConnectionStrings,
_options.TopicName, _options.ConsumerGroupName) {AckLevel = 0};
KafkaQueueAdapter adapter = new KafkaQueueAdapter(_streamQueueMapper, differentOptions, _providerName, factoryMock.Object, _logger);
Guid myGuid = Guid.NewGuid();
await adapter.QueueMessageBatchAsync(myGuid, "Test", new List<int>() { 1, 2, 3, 4 }, null, requestContext);
}
