Пример #1
0
        /// <summary> Init the factory.</summary>
        public virtual void Init(IProviderConfiguration config, string providerName, Logger logger)
        {
            if (config == null) throw new ArgumentNullException("config");
            if (!config.Properties.TryGetValue(DATA_CONNECTION_STRING, out dataConnectionString))
                throw new ArgumentException(String.Format("{0} property not set", DATA_CONNECTION_STRING));
            if (!config.Properties.TryGetValue(DEPLOYMENT_ID, out deploymentId))
                throw new ArgumentException(String.Format("{0} property not set", DEPLOYMENT_ID));
            
            string cacheSizeString;
            cacheSize = DEFAULT_CACHE_SIZE;
            if (config.Properties.TryGetValue(CACHE_SIZE_PARAM, out cacheSizeString))
            {
                if (!int.TryParse(cacheSizeString, out cacheSize))
                    throw new ArgumentException(String.Format("{0} invalid.  Must be int", CACHE_SIZE_PARAM));
            }
            string numQueuesString;
            numQueues = DEFAULT_NUM_QUEUES;
            if (config.Properties.TryGetValue(NUM_QUEUES_PARAM, out numQueuesString))
            {
                if (!int.TryParse(numQueuesString, out numQueues))
                    throw new ArgumentException(String.Format("{0} invalid.  Must be int", NUM_QUEUES_PARAM));
            }

            this.providerName = providerName;
            streamQueueMapper = new HashRingBasedStreamQueueMapper(numQueues, providerName);
            adapterCache = new SimpleQueueAdapterCache(this, cacheSize, logger);
        }
Пример #2
0
        /// <summary> Init the factory.</summary>
        public virtual void Init(IProviderConfiguration config, string providerName, Logger logger, IServiceProvider serviceProvider)
        {
            if (config == null) throw new ArgumentNullException("config");
            if (!config.Properties.TryGetValue(DataConnectionStringPropertyName, out dataConnectionString))
                throw new ArgumentException(String.Format("{0} property not set", DataConnectionStringPropertyName));
            if (!config.Properties.TryGetValue(DeploymentIdPropertyName, out deploymentId))
                throw new ArgumentException(String.Format("{0} property not set", DeploymentIdPropertyName));

            cacheSize = SimpleQueueAdapterCache.ParseSize(config, CacheSizeDefaultValue);

            string numQueuesString;
            numQueues = NumQueuesDefaultValue;
            if (config.Properties.TryGetValue(NumQueuesPropertyName, out numQueuesString))
            {
                if (!int.TryParse(numQueuesString, out numQueues))
                    throw new ArgumentException(String.Format("{0} invalid.  Must be int", NumQueuesPropertyName));
            }

            this.providerName = providerName;
            streamQueueMapper = new HashRingBasedStreamQueueMapper(numQueues, providerName);
            adapterCache = new SimpleQueueAdapterCache(cacheSize, logger);
            if (StreamFailureHandlerFactory == null)
            {
                StreamFailureHandlerFactory =
                    qid => Task.FromResult<IStreamFailureHandler>(new NoOpStreamDeliveryFailureHandler(false));
            }
        }
        public void Init(IProviderConfiguration config, string providerName, Logger logger, IServiceProvider serviceProvider)
        {
            if (config == null) throw new ArgumentNullException(nameof(config));
            if (logger == null) throw new ArgumentNullException(nameof(logger));
            if (String.IsNullOrEmpty(providerName)) throw new ArgumentNullException(nameof(providerName));

            // Creating an options object with all the config values
            _options = new KafkaStreamProviderOptions(config);

            if (!_options.UsingExternalMetrics)
            {
                Metric.Config.WithHttpEndpoint($"http://localhost:{_options.MetricsPort}/");
            }

            if (!_options.IncludeMetrics)
            {
                Metric.Context("KafkaStreamProvider").Advanced.CompletelyDisableMetrics();
            }

            _providerName = providerName;
            _streamQueueMapper = new HashRingBasedStreamQueueMapper(_options.NumOfQueues, providerName);
            _logger = logger;
            _adapter = new KafkaQueueAdapter(_streamQueueMapper, _options, providerName, new KafkaBatchFactory(), _logger);
            _adapterCache = new TimedQueueAdapterCache(this, TimeSpan.FromSeconds(_options.CacheTimespanInSeconds), _options.CacheSize, _options.CacheNumOfBuckets, logger);
        }
        public KafkaQueueAdapterUnitTests()
        {
            Mock<Logger> loggerMock = new Mock<Logger>();
            var connectionStrings = new List<Uri> {new Uri("http://kafka1:9092;http://kafka2:9092") };
            var topicName = "Jonathan.ab.KafkaStreamProviderTestsNew";
            var consumerGroupName = "TestConsumerGroupName";

            _logger = loggerMock.Object;
            _options = new KafkaStreamProviderOptions(connectionStrings.ToArray(), topicName, consumerGroupName);
            _streamQueueMapper = new HashRingBasedStreamQueueMapper(_options.NumOfQueues, _options.TopicName);
        }
        /// <summary>
        /// Helper method for testing.
        /// </summary>
        /// <param name="providerName"></param>
        /// <param name="deploymentId"></param>
        /// <param name="storageConnectionString"></param>
        /// <param name="logger"></param>
        /// <returns></returns>
        public static async Task DeleteAllUsedAzureQueues(string providerName, string deploymentId, string storageConnectionString, Logger logger)
        {
            if (deploymentId != null)
            {
                var queueMapper = new HashRingBasedStreamQueueMapper(AzureQueueAdapterFactory.DEFAULT_NUM_QUEUES, providerName);
                List<QueueId> allQueues = queueMapper.GetAllQueues().ToList();

                if (logger != null) logger.Info("About to delete all {0} Stream Queues\n", allQueues.Count);
                foreach (var queueId in allQueues)
                {
                    var manager = new AzureQueueDataManager(queueId.ToString(), deploymentId, storageConnectionString);
                    await manager.DeleteQueue();
                }
            }
        }
        /// <summary>
        /// Helper method for testing. Clears all messages in all the queues used by the specifed stream provider.
        /// </summary>
        /// <param name="providerName">The Azure Queue stream privider name.</param>
        /// <param name="deploymentId">The deployment ID hosting the stream provider.</param>
        /// <param name="storageConnectionString">The azure storage connection string.</param>
        public static async Task ClearAllUsedAzureQueues(string providerName, string deploymentId, string storageConnectionString)
        {
            if (deploymentId != null)
            {
                var queueMapper = new HashRingBasedStreamQueueMapper(AzureQueueAdapterFactory.NumQueuesDefaultValue, providerName);
                List<QueueId> allQueues = queueMapper.GetAllQueues().ToList();

                var deleteTasks = new List<Task>();
                foreach (var queueId in allQueues)
                {
                    var manager = new AzureQueueDataManager(queueId.ToString(), deploymentId, storageConnectionString);
                    deleteTasks.Add(manager.ClearQueue());
                }

                await Task.WhenAll(deleteTasks);
            }
        }
Пример #7
0
        /// <summary>
        /// Async method to delete all used queques, for specific provider and deploymentId
        /// </summary>
        /// <param name="providerName"></param>
        /// <param name="deploymentId"></param>
        /// <param name="storageConnectionString"></param>
        /// <returns> Task object for this async method </returns>
        public static async Task DeleteAllUsedQueues(string providerName, string deploymentId, string storageConnectionString)
        {
            if (deploymentId != null)
            {
                var queueMapper = new HashRingBasedStreamQueueMapper(SQSAdapterFactory.NumQueuesDefaultValue, providerName);
                List<QueueId> allQueues = queueMapper.GetAllQueues().ToList();

                var deleteTasks = new List<Task>();
                foreach (var queueId in allQueues)
                {
                    var manager = new SQSStorage(queueId.ToString(), storageConnectionString, deploymentId);
                    manager.InitQueueAsync().Wait();
                    deleteTasks.Add(manager.DeleteQueue());
                }

                await Task.WhenAll(deleteTasks);
            }
        }
        public void Init(IProviderConfiguration config, string providerName, Logger logger, IServiceProvider serviceProvider)
        {
            _logger = logger;
            _providerName = providerName;
            _config = config;
            _serviceProvider = serviceProvider;

            // Cache size
            string timeLimitString;
            _timeLimit = DefaultTimeLimit;
            if (config.Properties.TryGetValue(TimeLimitParam, out timeLimitString))
            {
                if (!int.TryParse(timeLimitString, out _timeLimit))
                    throw new ArgumentException($"{TimeLimitParam} invalid.  Must be int");
            }

            // # queues
            string numQueuesString;
            _numQueues = DefaultNumQueues;
            if (config.Properties.TryGetValue(NumQueuesParam, out numQueuesString))
            {
                if (!int.TryParse(numQueuesString, out _numQueues))
                    throw new ArgumentException($"{NumQueuesParam} invalid.  Must be int");
            }

            // Use Redis for queue?
            string useRedis;
            _useRedisForQueue = DefaultUseRedisForQueue;
            if (config.Properties.TryGetValue(UseRedisForQueueParam, out useRedis))
            {
                if (!bool.TryParse(useRedis, out _useRedisForQueue))
                    throw new ArgumentException($"{UseRedisForQueueParam} invalid value {useRedis}");
            }

            if (_useRedisForQueue)
                ReadRedisConnectionParams(config);

            _streamQueueMapper = new HashRingBasedStreamQueueMapper(_numQueues, providerName);
        }
        private Dictionary<SiloAddress, List<int>> GetQueueHistogram(Dictionary<SiloAddress, List<IRingRangeInternal>> siloRanges, int totalNumQueues)
        {
            HashRingBasedStreamQueueMapper queueMapper = new HashRingBasedStreamQueueMapper(totalNumQueues, "AzureQueues");
            var allQueues = queueMapper.GetAllQueues();

            Dictionary<SiloAddress, List<int>> queueHistogram = new Dictionary<SiloAddress, List<int>>();
            foreach (var siloRange in siloRanges)
            {
                List<int> agentRanges = new List<int>();
                foreach (IRingRangeInternal agentRange in siloRange.Value)
                {
                    int numQueues = queueMapper.GetQueuesForRange(agentRange).Count();
                    agentRanges.Add(numQueues);
                }
                agentRanges.Sort();
                queueHistogram.Add(siloRange.Key, agentRanges);
            }
            //queueHistogram.Sort((t1, t2) => t1.Item2.CompareTo(t2.Item2));
            return queueHistogram;
        }
 public void Init(IProviderConfiguration config, string providerName, Logger logger, IServiceProvider serviceProvider)
 {
     if (config == null) throw new ArgumentNullException(nameof(config));
     if (!config.Properties.TryGetValue(RABBITMQ_CONNECTION_STRING, out _dataConnectionString))
         throw new ArgumentException($"{RABBITMQ_CONNECTION_STRING} property not set");
     if (!config.Properties.TryGetValue(DEPLOYMENT_ID, out _deploymentId))
         throw new ArgumentException($"{DEPLOYMENT_ID} property not set");
     string cacheSizeString;
     _cacheSize = DEFAULT_CACHE_SIZE;
     if (config.Properties.TryGetValue(CACHE_SIZE_PARAM, out cacheSizeString))
         if (!int.TryParse(cacheSizeString, out _cacheSize))
             throw new ArgumentException($"{CACHE_SIZE_PARAM} invalid.  Must be int");
     string numQueuesString;
     _numQueues = DEFAULT_NUM_QUEUES;
     if (config.Properties.TryGetValue(NUM_QUEUES_PARAM, out numQueuesString))
         if (!int.TryParse(numQueuesString, out _numQueues))
             throw new ArgumentException($"{NUM_QUEUES_PARAM} invalid.  Must be int");
     _providerName = providerName;
     _streamQueueMapper = new HashRingBasedStreamQueueMapper(_numQueues, providerName);
     _adapterCache = new SimpleQueueAdapterCache(_cacheSize, logger);
 }
        public void QueueMessageBatchAsyncQueueingTwiceDifferentQueuesTest()
        {
            Mock<IKafkaBatchFactory> factoryMock = new Mock<IKafkaBatchFactory>();
            Dictionary<string, object> requestContext = new Dictionary<string, object>();

            factoryMock.Setup(x => x.ToKafkaMessage(It.IsAny<Guid>(), It.IsAny<string>(), It.IsAny<IEnumerable<int>>(), requestContext)).Returns(new Message() { Value = new byte[] { 0, 1, 2, 3 } });

            var twoQueuesStreamMapper = new HashRingBasedStreamQueueMapper(2, _options.TopicName);
            var twoQueuesOptions = new KafkaStreamProviderOptions(_options.ConnectionStrings, _options.TopicName, _options.ConsumerGroupName){NumOfQueues = 2};

            KafkaQueueAdapter adapter = new KafkaQueueAdapter(_streamQueueMapper, _options, _providerName, factoryMock.Object, _logger);            

            var first = Guid.NewGuid();
            var second = Guid.NewGuid();

            // Becuase we cannot mock the queue mapper.. we need to make sure we two guids that will return different queues...
            bool willGiveDifferentQueue = !(twoQueuesStreamMapper.GetQueueForStream(first, "test").Equals(twoQueuesStreamMapper.GetQueueForStream(second, "otherTest")));
            while (!willGiveDifferentQueue)
            {
                second = Guid.NewGuid();
                willGiveDifferentQueue = !(twoQueuesStreamMapper.GetQueueForStream(first, "test").Equals(twoQueuesStreamMapper.GetQueueForStream(second, "otherTest")));
            }

            Task.WaitAll(adapter.QueueMessageBatchAsync(first, "test", new List<int>() { 1, 2, 3, 4 }, null, requestContext),
                         adapter.QueueMessageBatchAsync(second, "otherTest", new List<int>() { 1, 2, 3, 4 }, null, requestContext));
        }