/// <summary> Init the factory.</summary>
public virtual void Init(IProviderConfiguration config, string providerName, Logger logger)
{
if (config == null) throw new ArgumentNullException("config");
if (!config.Properties.TryGetValue(DATA_CONNECTION_STRING, out dataConnectionString))
throw new ArgumentException(String.Format("{0} property not set", DATA_CONNECTION_STRING));
if (!config.Properties.TryGetValue(DEPLOYMENT_ID, out deploymentId))
throw new ArgumentException(String.Format("{0} property not set", DEPLOYMENT_ID));
string cacheSizeString;
cacheSize = DEFAULT_CACHE_SIZE;
if (config.Properties.TryGetValue(CACHE_SIZE_PARAM, out cacheSizeString))
{
if (!int.TryParse(cacheSizeString, out cacheSize))
throw new ArgumentException(String.Format("{0} invalid. Must be int", CACHE_SIZE_PARAM));
}
string numQueuesString;
numQueues = DEFAULT_NUM_QUEUES;
if (config.Properties.TryGetValue(NUM_QUEUES_PARAM, out numQueuesString))
{
if (!int.TryParse(numQueuesString, out numQueues))
throw new ArgumentException(String.Format("{0} invalid. Must be int", NUM_QUEUES_PARAM));
}
this.providerName = providerName;
streamQueueMapper = new HashRingBasedStreamQueueMapper(numQueues, providerName);
adapterCache = new SimpleQueueAdapterCache(this, cacheSize, logger);
}
/// <summary> Init the factory.</summary>
public virtual void Init(IProviderConfiguration config, string providerName, Logger logger, IServiceProvider serviceProvider)
{
if (config == null) throw new ArgumentNullException("config");
if (!config.Properties.TryGetValue(DataConnectionStringPropertyName, out dataConnectionString))
throw new ArgumentException(String.Format("{0} property not set", DataConnectionStringPropertyName));
if (!config.Properties.TryGetValue(DeploymentIdPropertyName, out deploymentId))
throw new ArgumentException(String.Format("{0} property not set", DeploymentIdPropertyName));
cacheSize = SimpleQueueAdapterCache.ParseSize(config, CacheSizeDefaultValue);
string numQueuesString;
numQueues = NumQueuesDefaultValue;
if (config.Properties.TryGetValue(NumQueuesPropertyName, out numQueuesString))
{
if (!int.TryParse(numQueuesString, out numQueues))
throw new ArgumentException(String.Format("{0} invalid. Must be int", NumQueuesPropertyName));
}
this.providerName = providerName;
streamQueueMapper = new HashRingBasedStreamQueueMapper(numQueues, providerName);
adapterCache = new SimpleQueueAdapterCache(cacheSize, logger);
if (StreamFailureHandlerFactory == null)
{
StreamFailureHandlerFactory =
qid => Task.FromResult<IStreamFailureHandler>(new NoOpStreamDeliveryFailureHandler(false));
}
}
public void Init(IProviderConfiguration config, string providerName, Logger logger, IServiceProvider serviceProvider)
{
if (config == null) throw new ArgumentNullException(nameof(config));
if (logger == null) throw new ArgumentNullException(nameof(logger));
if (String.IsNullOrEmpty(providerName)) throw new ArgumentNullException(nameof(providerName));
// Creating an options object with all the config values
_options = new KafkaStreamProviderOptions(config);
if (!_options.UsingExternalMetrics)
{
Metric.Config.WithHttpEndpoint($"http://localhost:{_options.MetricsPort}/");
}
if (!_options.IncludeMetrics)
{
Metric.Context("KafkaStreamProvider").Advanced.CompletelyDisableMetrics();
}
_providerName = providerName;
_streamQueueMapper = new HashRingBasedStreamQueueMapper(_options.NumOfQueues, providerName);
_logger = logger;
_adapter = new KafkaQueueAdapter(_streamQueueMapper, _options, providerName, new KafkaBatchFactory(), _logger);
_adapterCache = new TimedQueueAdapterCache(this, TimeSpan.FromSeconds(_options.CacheTimespanInSeconds), _options.CacheSize, _options.CacheNumOfBuckets, logger);
}
public KafkaQueueAdapterUnitTests()
{
Mock<Logger> loggerMock = new Mock<Logger>();
var connectionStrings = new List<Uri> {new Uri("http://kafka1:9092;http://kafka2:9092") };
var topicName = "Jonathan.ab.KafkaStreamProviderTestsNew";
var consumerGroupName = "TestConsumerGroupName";
_logger = loggerMock.Object;
_options = new KafkaStreamProviderOptions(connectionStrings.ToArray(), topicName, consumerGroupName);
_streamQueueMapper = new HashRingBasedStreamQueueMapper(_options.NumOfQueues, _options.TopicName);
}
/// <summary>
/// Helper method for testing.
/// </summary>
/// <param name="providerName"></param>
/// <param name="deploymentId"></param>
/// <param name="storageConnectionString"></param>
/// <param name="logger"></param>
/// <returns></returns>
public static async Task DeleteAllUsedAzureQueues(string providerName, string deploymentId, string storageConnectionString, Logger logger)
{
if (deploymentId != null)
{
var queueMapper = new HashRingBasedStreamQueueMapper(AzureQueueAdapterFactory.DEFAULT_NUM_QUEUES, providerName);
List<QueueId> allQueues = queueMapper.GetAllQueues().ToList();
if (logger != null) logger.Info("About to delete all {0} Stream Queues\n", allQueues.Count);
foreach (var queueId in allQueues)
{
var manager = new AzureQueueDataManager(queueId.ToString(), deploymentId, storageConnectionString);
await manager.DeleteQueue();
}
}
}
/// <summary>
/// Helper method for testing. Clears all messages in all the queues used by the specifed stream provider.
/// </summary>
/// <param name="providerName">The Azure Queue stream privider name.</param>
/// <param name="deploymentId">The deployment ID hosting the stream provider.</param>
/// <param name="storageConnectionString">The azure storage connection string.</param>
public static async Task ClearAllUsedAzureQueues(string providerName, string deploymentId, string storageConnectionString)
{
if (deploymentId != null)
{
var queueMapper = new HashRingBasedStreamQueueMapper(AzureQueueAdapterFactory.NumQueuesDefaultValue, providerName);
List<QueueId> allQueues = queueMapper.GetAllQueues().ToList();
var deleteTasks = new List<Task>();
foreach (var queueId in allQueues)
{
var manager = new AzureQueueDataManager(queueId.ToString(), deploymentId, storageConnectionString);
deleteTasks.Add(manager.ClearQueue());
}
await Task.WhenAll(deleteTasks);
}
}
/// <summary>
/// Async method to delete all used queques, for specific provider and deploymentId
/// </summary>
/// <param name="providerName"></param>
/// <param name="deploymentId"></param>
/// <param name="storageConnectionString"></param>
/// <returns> Task object for this async method </returns>
public static async Task DeleteAllUsedQueues(string providerName, string deploymentId, string storageConnectionString)
{
if (deploymentId != null)
{
var queueMapper = new HashRingBasedStreamQueueMapper(SQSAdapterFactory.NumQueuesDefaultValue, providerName);
List<QueueId> allQueues = queueMapper.GetAllQueues().ToList();
var deleteTasks = new List<Task>();
foreach (var queueId in allQueues)
{
var manager = new SQSStorage(queueId.ToString(), storageConnectionString, deploymentId);
manager.InitQueueAsync().Wait();
deleteTasks.Add(manager.DeleteQueue());
}
await Task.WhenAll(deleteTasks);
}
}
public void Init(IProviderConfiguration config, string providerName, Logger logger, IServiceProvider serviceProvider)
{
_logger = logger;
_providerName = providerName;
_config = config;
_serviceProvider = serviceProvider;
// Cache size
string timeLimitString;
_timeLimit = DefaultTimeLimit;
if (config.Properties.TryGetValue(TimeLimitParam, out timeLimitString))
{
if (!int.TryParse(timeLimitString, out _timeLimit))
throw new ArgumentException($"{TimeLimitParam} invalid. Must be int");
}
// # queues
string numQueuesString;
_numQueues = DefaultNumQueues;
if (config.Properties.TryGetValue(NumQueuesParam, out numQueuesString))
{
if (!int.TryParse(numQueuesString, out _numQueues))
throw new ArgumentException($"{NumQueuesParam} invalid. Must be int");
}
// Use Redis for queue?
string useRedis;
_useRedisForQueue = DefaultUseRedisForQueue;
if (config.Properties.TryGetValue(UseRedisForQueueParam, out useRedis))
{
if (!bool.TryParse(useRedis, out _useRedisForQueue))
throw new ArgumentException($"{UseRedisForQueueParam} invalid value {useRedis}");
}
if (_useRedisForQueue)
ReadRedisConnectionParams(config);
_streamQueueMapper = new HashRingBasedStreamQueueMapper(_numQueues, providerName);
}
private Dictionary<SiloAddress, List<int>> GetQueueHistogram(Dictionary<SiloAddress, List<IRingRangeInternal>> siloRanges, int totalNumQueues)
{
HashRingBasedStreamQueueMapper queueMapper = new HashRingBasedStreamQueueMapper(totalNumQueues, "AzureQueues");
var allQueues = queueMapper.GetAllQueues();
Dictionary<SiloAddress, List<int>> queueHistogram = new Dictionary<SiloAddress, List<int>>();
foreach (var siloRange in siloRanges)
{
List<int> agentRanges = new List<int>();
foreach (IRingRangeInternal agentRange in siloRange.Value)
{
int numQueues = queueMapper.GetQueuesForRange(agentRange).Count();
agentRanges.Add(numQueues);
}
agentRanges.Sort();
queueHistogram.Add(siloRange.Key, agentRanges);
}
//queueHistogram.Sort((t1, t2) => t1.Item2.CompareTo(t2.Item2));
return queueHistogram;
}
public void Init(IProviderConfiguration config, string providerName, Logger logger, IServiceProvider serviceProvider)
{
if (config == null) throw new ArgumentNullException(nameof(config));
if (!config.Properties.TryGetValue(RABBITMQ_CONNECTION_STRING, out _dataConnectionString))
throw new ArgumentException($"{RABBITMQ_CONNECTION_STRING} property not set");
if (!config.Properties.TryGetValue(DEPLOYMENT_ID, out _deploymentId))
throw new ArgumentException($"{DEPLOYMENT_ID} property not set");
string cacheSizeString;
_cacheSize = DEFAULT_CACHE_SIZE;
if (config.Properties.TryGetValue(CACHE_SIZE_PARAM, out cacheSizeString))
if (!int.TryParse(cacheSizeString, out _cacheSize))
throw new ArgumentException($"{CACHE_SIZE_PARAM} invalid. Must be int");
string numQueuesString;
_numQueues = DEFAULT_NUM_QUEUES;
if (config.Properties.TryGetValue(NUM_QUEUES_PARAM, out numQueuesString))
if (!int.TryParse(numQueuesString, out _numQueues))
throw new ArgumentException($"{NUM_QUEUES_PARAM} invalid. Must be int");
_providerName = providerName;
_streamQueueMapper = new HashRingBasedStreamQueueMapper(_numQueues, providerName);
_adapterCache = new SimpleQueueAdapterCache(_cacheSize, logger);
}
public void QueueMessageBatchAsyncQueueingTwiceDifferentQueuesTest()
{
Mock<IKafkaBatchFactory> factoryMock = new Mock<IKafkaBatchFactory>();
Dictionary<string, object> requestContext = new Dictionary<string, object>();
factoryMock.Setup(x => x.ToKafkaMessage(It.IsAny<Guid>(), It.IsAny<string>(), It.IsAny<IEnumerable<int>>(), requestContext)).Returns(new Message() { Value = new byte[] { 0, 1, 2, 3 } });
var twoQueuesStreamMapper = new HashRingBasedStreamQueueMapper(2, _options.TopicName);
var twoQueuesOptions = new KafkaStreamProviderOptions(_options.ConnectionStrings, _options.TopicName, _options.ConsumerGroupName){NumOfQueues = 2};
KafkaQueueAdapter adapter = new KafkaQueueAdapter(_streamQueueMapper, _options, _providerName, factoryMock.Object, _logger);
var first = Guid.NewGuid();
var second = Guid.NewGuid();
// Becuase we cannot mock the queue mapper.. we need to make sure we two guids that will return different queues...
bool willGiveDifferentQueue = !(twoQueuesStreamMapper.GetQueueForStream(first, "test").Equals(twoQueuesStreamMapper.GetQueueForStream(second, "otherTest")));
while (!willGiveDifferentQueue)
{
second = Guid.NewGuid();
willGiveDifferentQueue = !(twoQueuesStreamMapper.GetQueueForStream(first, "test").Equals(twoQueuesStreamMapper.GetQueueForStream(second, "otherTest")));
}
Task.WaitAll(adapter.QueueMessageBatchAsync(first, "test", new List<int>() { 1, 2, 3, 4 }, null, requestContext),
adapter.QueueMessageBatchAsync(second, "otherTest", new List<int>() { 1, 2, 3, 4 }, null, requestContext));
}