/// <summary> Init the factory.</summary> public virtual void Init(IProviderConfiguration config, string providerName, Logger logger) { if (config == null) throw new ArgumentNullException("config"); if (!config.Properties.TryGetValue(DATA_CONNECTION_STRING, out dataConnectionString)) throw new ArgumentException(String.Format("{0} property not set", DATA_CONNECTION_STRING)); if (!config.Properties.TryGetValue(DEPLOYMENT_ID, out deploymentId)) throw new ArgumentException(String.Format("{0} property not set", DEPLOYMENT_ID)); string cacheSizeString; cacheSize = DEFAULT_CACHE_SIZE; if (config.Properties.TryGetValue(CACHE_SIZE_PARAM, out cacheSizeString)) { if (!int.TryParse(cacheSizeString, out cacheSize)) throw new ArgumentException(String.Format("{0} invalid. Must be int", CACHE_SIZE_PARAM)); } string numQueuesString; numQueues = DEFAULT_NUM_QUEUES; if (config.Properties.TryGetValue(NUM_QUEUES_PARAM, out numQueuesString)) { if (!int.TryParse(numQueuesString, out numQueues)) throw new ArgumentException(String.Format("{0} invalid. Must be int", NUM_QUEUES_PARAM)); } this.providerName = providerName; streamQueueMapper = new HashRingBasedStreamQueueMapper(numQueues, providerName); adapterCache = new SimpleQueueAdapterCache(this, cacheSize, logger); }
/// <summary> Init the factory.</summary> public virtual void Init(IProviderConfiguration config, string providerName, Logger logger, IServiceProvider serviceProvider) { if (config == null) throw new ArgumentNullException("config"); if (!config.Properties.TryGetValue(DataConnectionStringPropertyName, out dataConnectionString)) throw new ArgumentException(String.Format("{0} property not set", DataConnectionStringPropertyName)); if (!config.Properties.TryGetValue(DeploymentIdPropertyName, out deploymentId)) throw new ArgumentException(String.Format("{0} property not set", DeploymentIdPropertyName)); cacheSize = SimpleQueueAdapterCache.ParseSize(config, CacheSizeDefaultValue); string numQueuesString; numQueues = NumQueuesDefaultValue; if (config.Properties.TryGetValue(NumQueuesPropertyName, out numQueuesString)) { if (!int.TryParse(numQueuesString, out numQueues)) throw new ArgumentException(String.Format("{0} invalid. Must be int", NumQueuesPropertyName)); } this.providerName = providerName; streamQueueMapper = new HashRingBasedStreamQueueMapper(numQueues, providerName); adapterCache = new SimpleQueueAdapterCache(cacheSize, logger); if (StreamFailureHandlerFactory == null) { StreamFailureHandlerFactory = qid => Task.FromResult<IStreamFailureHandler>(new NoOpStreamDeliveryFailureHandler(false)); } }
public void Init(IProviderConfiguration config, string providerName, Logger logger, IServiceProvider serviceProvider) { if (config == null) throw new ArgumentNullException(nameof(config)); if (logger == null) throw new ArgumentNullException(nameof(logger)); if (String.IsNullOrEmpty(providerName)) throw new ArgumentNullException(nameof(providerName)); // Creating an options object with all the config values _options = new KafkaStreamProviderOptions(config); if (!_options.UsingExternalMetrics) { Metric.Config.WithHttpEndpoint($"http://localhost:{_options.MetricsPort}/"); } if (!_options.IncludeMetrics) { Metric.Context("KafkaStreamProvider").Advanced.CompletelyDisableMetrics(); } _providerName = providerName; _streamQueueMapper = new HashRingBasedStreamQueueMapper(_options.NumOfQueues, providerName); _logger = logger; _adapter = new KafkaQueueAdapter(_streamQueueMapper, _options, providerName, new KafkaBatchFactory(), _logger); _adapterCache = new TimedQueueAdapterCache(this, TimeSpan.FromSeconds(_options.CacheTimespanInSeconds), _options.CacheSize, _options.CacheNumOfBuckets, logger); }
public KafkaQueueAdapterUnitTests() { Mock<Logger> loggerMock = new Mock<Logger>(); var connectionStrings = new List<Uri> {new Uri("http://kafka1:9092;http://kafka2:9092") }; var topicName = "Jonathan.ab.KafkaStreamProviderTestsNew"; var consumerGroupName = "TestConsumerGroupName"; _logger = loggerMock.Object; _options = new KafkaStreamProviderOptions(connectionStrings.ToArray(), topicName, consumerGroupName); _streamQueueMapper = new HashRingBasedStreamQueueMapper(_options.NumOfQueues, _options.TopicName); }
/// <summary> /// Helper method for testing. /// </summary> /// <param name="providerName"></param> /// <param name="deploymentId"></param> /// <param name="storageConnectionString"></param> /// <param name="logger"></param> /// <returns></returns> public static async Task DeleteAllUsedAzureQueues(string providerName, string deploymentId, string storageConnectionString, Logger logger) { if (deploymentId != null) { var queueMapper = new HashRingBasedStreamQueueMapper(AzureQueueAdapterFactory.DEFAULT_NUM_QUEUES, providerName); List<QueueId> allQueues = queueMapper.GetAllQueues().ToList(); if (logger != null) logger.Info("About to delete all {0} Stream Queues\n", allQueues.Count); foreach (var queueId in allQueues) { var manager = new AzureQueueDataManager(queueId.ToString(), deploymentId, storageConnectionString); await manager.DeleteQueue(); } } }
/// <summary> /// Helper method for testing. Clears all messages in all the queues used by the specifed stream provider. /// </summary> /// <param name="providerName">The Azure Queue stream privider name.</param> /// <param name="deploymentId">The deployment ID hosting the stream provider.</param> /// <param name="storageConnectionString">The azure storage connection string.</param> public static async Task ClearAllUsedAzureQueues(string providerName, string deploymentId, string storageConnectionString) { if (deploymentId != null) { var queueMapper = new HashRingBasedStreamQueueMapper(AzureQueueAdapterFactory.NumQueuesDefaultValue, providerName); List<QueueId> allQueues = queueMapper.GetAllQueues().ToList(); var deleteTasks = new List<Task>(); foreach (var queueId in allQueues) { var manager = new AzureQueueDataManager(queueId.ToString(), deploymentId, storageConnectionString); deleteTasks.Add(manager.ClearQueue()); } await Task.WhenAll(deleteTasks); } }
/// <summary> /// Async method to delete all used queques, for specific provider and deploymentId /// </summary> /// <param name="providerName"></param> /// <param name="deploymentId"></param> /// <param name="storageConnectionString"></param> /// <returns> Task object for this async method </returns> public static async Task DeleteAllUsedQueues(string providerName, string deploymentId, string storageConnectionString) { if (deploymentId != null) { var queueMapper = new HashRingBasedStreamQueueMapper(SQSAdapterFactory.NumQueuesDefaultValue, providerName); List<QueueId> allQueues = queueMapper.GetAllQueues().ToList(); var deleteTasks = new List<Task>(); foreach (var queueId in allQueues) { var manager = new SQSStorage(queueId.ToString(), storageConnectionString, deploymentId); manager.InitQueueAsync().Wait(); deleteTasks.Add(manager.DeleteQueue()); } await Task.WhenAll(deleteTasks); } }
public void Init(IProviderConfiguration config, string providerName, Logger logger, IServiceProvider serviceProvider) { _logger = logger; _providerName = providerName; _config = config; _serviceProvider = serviceProvider; // Cache size string timeLimitString; _timeLimit = DefaultTimeLimit; if (config.Properties.TryGetValue(TimeLimitParam, out timeLimitString)) { if (!int.TryParse(timeLimitString, out _timeLimit)) throw new ArgumentException($"{TimeLimitParam} invalid. Must be int"); } // # queues string numQueuesString; _numQueues = DefaultNumQueues; if (config.Properties.TryGetValue(NumQueuesParam, out numQueuesString)) { if (!int.TryParse(numQueuesString, out _numQueues)) throw new ArgumentException($"{NumQueuesParam} invalid. Must be int"); } // Use Redis for queue? string useRedis; _useRedisForQueue = DefaultUseRedisForQueue; if (config.Properties.TryGetValue(UseRedisForQueueParam, out useRedis)) { if (!bool.TryParse(useRedis, out _useRedisForQueue)) throw new ArgumentException($"{UseRedisForQueueParam} invalid value {useRedis}"); } if (_useRedisForQueue) ReadRedisConnectionParams(config); _streamQueueMapper = new HashRingBasedStreamQueueMapper(_numQueues, providerName); }
private Dictionary<SiloAddress, List<int>> GetQueueHistogram(Dictionary<SiloAddress, List<IRingRangeInternal>> siloRanges, int totalNumQueues) { HashRingBasedStreamQueueMapper queueMapper = new HashRingBasedStreamQueueMapper(totalNumQueues, "AzureQueues"); var allQueues = queueMapper.GetAllQueues(); Dictionary<SiloAddress, List<int>> queueHistogram = new Dictionary<SiloAddress, List<int>>(); foreach (var siloRange in siloRanges) { List<int> agentRanges = new List<int>(); foreach (IRingRangeInternal agentRange in siloRange.Value) { int numQueues = queueMapper.GetQueuesForRange(agentRange).Count(); agentRanges.Add(numQueues); } agentRanges.Sort(); queueHistogram.Add(siloRange.Key, agentRanges); } //queueHistogram.Sort((t1, t2) => t1.Item2.CompareTo(t2.Item2)); return queueHistogram; }
public void Init(IProviderConfiguration config, string providerName, Logger logger, IServiceProvider serviceProvider) { if (config == null) throw new ArgumentNullException(nameof(config)); if (!config.Properties.TryGetValue(RABBITMQ_CONNECTION_STRING, out _dataConnectionString)) throw new ArgumentException($"{RABBITMQ_CONNECTION_STRING} property not set"); if (!config.Properties.TryGetValue(DEPLOYMENT_ID, out _deploymentId)) throw new ArgumentException($"{DEPLOYMENT_ID} property not set"); string cacheSizeString; _cacheSize = DEFAULT_CACHE_SIZE; if (config.Properties.TryGetValue(CACHE_SIZE_PARAM, out cacheSizeString)) if (!int.TryParse(cacheSizeString, out _cacheSize)) throw new ArgumentException($"{CACHE_SIZE_PARAM} invalid. Must be int"); string numQueuesString; _numQueues = DEFAULT_NUM_QUEUES; if (config.Properties.TryGetValue(NUM_QUEUES_PARAM, out numQueuesString)) if (!int.TryParse(numQueuesString, out _numQueues)) throw new ArgumentException($"{NUM_QUEUES_PARAM} invalid. Must be int"); _providerName = providerName; _streamQueueMapper = new HashRingBasedStreamQueueMapper(_numQueues, providerName); _adapterCache = new SimpleQueueAdapterCache(_cacheSize, logger); }
public void QueueMessageBatchAsyncQueueingTwiceDifferentQueuesTest() { Mock<IKafkaBatchFactory> factoryMock = new Mock<IKafkaBatchFactory>(); Dictionary<string, object> requestContext = new Dictionary<string, object>(); factoryMock.Setup(x => x.ToKafkaMessage(It.IsAny<Guid>(), It.IsAny<string>(), It.IsAny<IEnumerable<int>>(), requestContext)).Returns(new Message() { Value = new byte[] { 0, 1, 2, 3 } }); var twoQueuesStreamMapper = new HashRingBasedStreamQueueMapper(2, _options.TopicName); var twoQueuesOptions = new KafkaStreamProviderOptions(_options.ConnectionStrings, _options.TopicName, _options.ConsumerGroupName){NumOfQueues = 2}; KafkaQueueAdapter adapter = new KafkaQueueAdapter(_streamQueueMapper, _options, _providerName, factoryMock.Object, _logger); var first = Guid.NewGuid(); var second = Guid.NewGuid(); // Becuase we cannot mock the queue mapper.. we need to make sure we two guids that will return different queues... bool willGiveDifferentQueue = !(twoQueuesStreamMapper.GetQueueForStream(first, "test").Equals(twoQueuesStreamMapper.GetQueueForStream(second, "otherTest"))); while (!willGiveDifferentQueue) { second = Guid.NewGuid(); willGiveDifferentQueue = !(twoQueuesStreamMapper.GetQueueForStream(first, "test").Equals(twoQueuesStreamMapper.GetQueueForStream(second, "otherTest"))); } Task.WaitAll(adapter.QueueMessageBatchAsync(first, "test", new List<int>() { 1, 2, 3, 4 }, null, requestContext), adapter.QueueMessageBatchAsync(second, "otherTest", new List<int>() { 1, 2, 3, 4 }, null, requestContext)); }