/// <summary> Init the factory.</summary> public virtual void Init() { this.streamQueueMapper = new HashRingBasedStreamQueueMapper(this.options.NumQueues, providerName); this.adapterCache = new SimpleQueueAdapterCache(this.options.CacheSize, this.providerName, this.loggerFactory); this.StreamFailureHandlerFactory = this.StreamFailureHandlerFactory ?? ((qid) => Task.FromResult <IStreamFailureHandler>(new NoOpStreamDeliveryFailureHandler())); }
/// <summary> Init the factory.</summary> public virtual void Init(IProviderConfiguration config, string providerName, Logger logger, IServiceProvider serviceProvider) { if (config == null) throw new ArgumentNullException("config"); if (!config.Properties.TryGetValue(DataConnectionStringPropertyName, out dataConnectionString)) throw new ArgumentException(String.Format("{0} property not set", DataConnectionStringPropertyName)); if (!config.Properties.TryGetValue(DeploymentIdPropertyName, out deploymentId)) throw new ArgumentException(String.Format("{0} property not set", DeploymentIdPropertyName)); cacheSize = SimpleQueueAdapterCache.ParseSize(config, CacheSizeDefaultValue); string numQueuesString; numQueues = NumQueuesDefaultValue; if (config.Properties.TryGetValue(NumQueuesPropertyName, out numQueuesString)) { if (!int.TryParse(numQueuesString, out numQueues)) throw new ArgumentException(String.Format("{0} invalid. Must be int", NumQueuesPropertyName)); } this.providerName = providerName; streamQueueMapper = new HashRingBasedStreamQueueMapper(numQueues, providerName); adapterCache = new SimpleQueueAdapterCache(cacheSize, logger); if (StreamFailureHandlerFactory == null) { StreamFailureHandlerFactory = qid => Task.FromResult<IStreamFailureHandler>(new NoOpStreamDeliveryFailureHandler(false)); } }
/// <summary> Init the factory.</summary> public virtual void Init(IProviderConfiguration config, string providerName, Logger logger) { if (config == null) throw new ArgumentNullException("config"); if (!config.Properties.TryGetValue(DATA_CONNECTION_STRING, out dataConnectionString)) throw new ArgumentException(String.Format("{0} property not set", DATA_CONNECTION_STRING)); if (!config.Properties.TryGetValue(DEPLOYMENT_ID, out deploymentId)) throw new ArgumentException(String.Format("{0} property not set", DEPLOYMENT_ID)); string cacheSizeString; cacheSize = DEFAULT_CACHE_SIZE; if (config.Properties.TryGetValue(CACHE_SIZE_PARAM, out cacheSizeString)) { if (!int.TryParse(cacheSizeString, out cacheSize)) throw new ArgumentException(String.Format("{0} invalid. Must be int", CACHE_SIZE_PARAM)); } string numQueuesString; numQueues = DEFAULT_NUM_QUEUES; if (config.Properties.TryGetValue(NUM_QUEUES_PARAM, out numQueuesString)) { if (!int.TryParse(numQueuesString, out numQueues)) throw new ArgumentException(String.Format("{0} invalid. Must be int", NUM_QUEUES_PARAM)); } this.providerName = providerName; streamQueueMapper = new HashRingBasedStreamQueueMapper(numQueues, providerName); adapterCache = new SimpleQueueAdapterCache(this, cacheSize, logger); }
public void Init(IProviderConfiguration config, string providerName, Logger logger, IServiceProvider serviceProvider) { if (config == null) { throw new ArgumentNullException(nameof(config)); } if (logger == null) { throw new ArgumentNullException(nameof(logger)); } if (String.IsNullOrEmpty(providerName)) { throw new ArgumentNullException(nameof(providerName)); } // Creating an options object with all the config values _options = new KafkaStreamProviderOptions(config); if (!_options.UsingExternalMetrics) { Metric.Config.WithHttpEndpoint($"http://localhost:{_options.MetricsPort}/"); } if (!_options.IncludeMetrics) { Metric.Context("KafkaStreamProvider").Advanced.CompletelyDisableMetrics(); } _providerName = providerName; _streamQueueMapper = new HashRingBasedStreamQueueMapper(_options.NumOfQueues, providerName); _logger = logger; _adapter = new KafkaQueueAdapter(_streamQueueMapper, _options, providerName, new KafkaBatchFactory(), _logger); _adapterCache = new TimedQueueAdapterCache(this, TimeSpan.FromSeconds(_options.CacheTimespanInSeconds), _options.CacheSize, _options.CacheNumOfBuckets, logger); }
/// <summary> Init the factory.</summary> public virtual void Init(IProviderConfiguration config, string providerName, Logger logger, IServiceProvider serviceProvider) { if (config == null) { throw new ArgumentNullException("config"); } if (!config.Properties.TryGetValue(DataConnectionStringPropertyName, out dataConnectionString)) { throw new ArgumentException(string.Format("{0} property not set", DataConnectionStringPropertyName)); } if (!config.Properties.TryGetValue(DeploymentIdPropertyName, out deploymentId)) { throw new ArgumentException(string.Format("{0} property not set", DeploymentIdPropertyName)); } cacheSize = SimpleQueueAdapterCache.ParseSize(config, CacheSizeDefaultValue); numQueues = config.GetIntProperty(NumQueuesPropertyName, NumQueuesDefaultValue); this.providerName = providerName; streamQueueMapper = new HashRingBasedStreamQueueMapper(numQueues, providerName); adapterCache = new SimpleQueueAdapterCache(cacheSize, logger); if (StreamFailureHandlerFactory == null) { StreamFailureHandlerFactory = qid => Task.FromResult <IStreamFailureHandler>(new NoOpStreamDeliveryFailureHandler(false)); } }
/// <summary> Init the factory.</summary> public virtual void Init(IProviderConfiguration config, string providerName, Logger logger, IServiceProvider serviceProvider) { if (config == null) { throw new ArgumentNullException("config"); } if (!config.Properties.TryGetValue(DATA_CONNECTION_STRING, out dataConnectionString)) { throw new ArgumentException(String.Format("{0} property not set", DATA_CONNECTION_STRING)); } if (!config.Properties.TryGetValue(DEPLOYMENT_ID, out deploymentId)) { throw new ArgumentException(String.Format("{0} property not set", DEPLOYMENT_ID)); } cacheSize = SimpleQueueAdapterCache.ParseSize(config, DEFAULT_CACHE_SIZE); string numQueuesString; numQueues = DEFAULT_NUM_QUEUES; if (config.Properties.TryGetValue(NUM_QUEUES_PARAM, out numQueuesString)) { if (!int.TryParse(numQueuesString, out numQueues)) { throw new ArgumentException(String.Format("{0} invalid. Must be int", NUM_QUEUES_PARAM)); } } this.providerName = providerName; streamQueueMapper = new HashRingBasedStreamQueueMapper(numQueues, providerName); adapterCache = new SimpleQueueAdapterCache(cacheSize, logger); }
/// <summary> Init the factory.</summary> public virtual void Init(IProviderConfiguration config, string providerName, Logger logger, IServiceProvider serviceProvider) { if (config == null) { throw new ArgumentNullException("config"); } if (!config.Properties.TryGetValue(DataConnectionStringPropertyName, out dataConnectionString)) { throw new ArgumentException(String.Format("{0} property not set", DataConnectionStringPropertyName)); } if (!config.Properties.TryGetValue(DeploymentIdPropertyName, out deploymentId)) { throw new ArgumentException(String.Format("{0} property not set", DeploymentIdPropertyName)); } cacheSize = SimpleQueueAdapterCache.ParseSize(config, CacheSizeDefaultValue); string numQueuesString; numQueues = NumQueuesDefaultValue; if (config.Properties.TryGetValue(NumQueuesPropertyName, out numQueuesString)) { if (!int.TryParse(numQueuesString, out numQueues)) { throw new ArgumentException(String.Format("{0} invalid. Must be int", NumQueuesPropertyName)); } } this.providerName = providerName; streamQueueMapper = new HashRingBasedStreamQueueMapper(numQueues, providerName); adapterCache = new SimpleQueueAdapterCache(cacheSize, logger); }
public void Init(IProviderConfiguration config, string providerName, Logger logger, IServiceProvider serviceProvider) { if (config == null) throw new ArgumentNullException(nameof(config)); if (logger == null) throw new ArgumentNullException(nameof(logger)); if (String.IsNullOrEmpty(providerName)) throw new ArgumentNullException(nameof(providerName)); // Creating an options object with all the config values _options = new KafkaStreamProviderOptions(config); if (!_options.UsingExternalMetrics) { Metric.Config.WithHttpEndpoint($"http://localhost:{_options.MetricsPort}/"); } if (!_options.IncludeMetrics) { Metric.Context("KafkaStreamProvider").Advanced.CompletelyDisableMetrics(); } _providerName = providerName; _streamQueueMapper = new HashRingBasedStreamQueueMapper(_options.NumOfQueues, providerName); _logger = logger; _adapter = new KafkaQueueAdapter(_streamQueueMapper, _options, providerName, new KafkaBatchFactory(), _logger); _adapterCache = new TimedQueueAdapterCache(this, TimeSpan.FromSeconds(_options.CacheTimespanInSeconds), _options.CacheSize, _options.CacheNumOfBuckets, logger); }
public KafkaAdapterFactory( string name, KafkaStreamOptions options, SimpleQueueCacheOptions cacheOptions, SerializationManager serializationManager, ILoggerFactory loggerFactory, IGrainFactory grainFactory ) { _options = options ?? throw new ArgumentNullException(nameof(options)); _name = name; _serializationManager = serializationManager; _loggerFactory = loggerFactory; _grainFactory = grainFactory; _logger = loggerFactory.CreateLogger <KafkaAdapterFactory>(); if (options.Topics != null && options.Topics.Count == 0) { throw new ArgumentNullException(nameof(options.Topics)); } _adapterCache = new SimpleQueueAdapterCache( cacheOptions, name, loggerFactory ); _queueProperties = GetQueuesProperties(); _streamQueueMapper = new ExternalQueueMapper(_queueProperties.Values); }
public void Init(IProviderConfiguration config, string providerName, Logger logger, IServiceProvider serviceProvider) { if (config == null) { throw new ArgumentNullException(nameof(config)); } if (providerName == null) { throw new ArgumentNullException(nameof(providerName)); } if (logger == null) { throw new ArgumentNullException(nameof(logger)); } if (serviceProvider == null) { throw new ArgumentNullException(nameof(serviceProvider)); } m_ProviderName = providerName; m_Logger = logger; m_QueueMapper = new EventStoreQueueMapper(1, providerName); m_QueueAdapterCache = new EventStoreQueueAdapterCache(this, m_Logger); m_ConnectionString = config.Properties["ConnectionString"]; CreateConnection(); }
/// <summary> Init the factory.</summary> public virtual void Init(IProviderConfiguration config, string providerName, Logger logger, IServiceProvider serviceProvider) { if (config == null) { throw new ArgumentNullException(nameof(config)); } if (!config.Properties.TryGetValue(AzureQueueAdapterConstants.DataConnectionStringPropertyName, out dataConnectionString)) { throw new ArgumentException($"{AzureQueueAdapterConstants.DataConnectionStringPropertyName} property not set"); } if (!config.Properties.TryGetValue(AzureQueueAdapterConstants.DeploymentIdPropertyName, out deploymentId)) { throw new ArgumentException($"{AzureQueueAdapterConstants.DeploymentIdPropertyName} property not set"); } string messageVisibilityTimeoutRaw; if (config.Properties.TryGetValue(AzureQueueAdapterConstants.MessageVisibilityTimeoutPropertyName, out messageVisibilityTimeoutRaw)) { TimeSpan messageVisibilityTimeoutTemp; if (!TimeSpan.TryParse(messageVisibilityTimeoutRaw, out messageVisibilityTimeoutTemp)) { throw new ArgumentException( $"Failed to parse {AzureQueueAdapterConstants.MessageVisibilityTimeoutPropertyName} value '{messageVisibilityTimeoutRaw}' as a TimeSpan"); } messageVisibilityTimeout = messageVisibilityTimeoutTemp; } else { messageVisibilityTimeout = null; } cacheSize = SimpleQueueAdapterCache.ParseSize(config, AzureQueueAdapterConstants.CacheSizeDefaultValue); this.loggerFactory = serviceProvider.GetRequiredService <ILoggerFactory>(); string numQueuesString; numQueues = AzureQueueAdapterConstants.NumQueuesDefaultValue; if (config.Properties.TryGetValue(AzureQueueAdapterConstants.NumQueuesPropertyName, out numQueuesString)) { if (!int.TryParse(numQueuesString, out numQueues)) { throw new ArgumentException($"{AzureQueueAdapterConstants.NumQueuesPropertyName} invalid. Must be int"); } } this.providerName = providerName; streamQueueMapper = new HashRingBasedStreamQueueMapper(numQueues, providerName); adapterCache = new SimpleQueueAdapterCache(cacheSize, logger); if (StreamFailureHandlerFactory == null) { StreamFailureHandlerFactory = qid => Task.FromResult <IStreamFailureHandler>(new NoOpStreamDeliveryFailureHandler()); } this.SerializationManager = serviceProvider.GetRequiredService <SerializationManager>(); this.adaptorFactory = () => ActivatorUtilities.GetServiceOrCreateInstance <TDataAdapter>(serviceProvider); }
public virtual void Init() { this._streamQueueMapper = new HashRingBasedStreamQueueMapper(this.options.NumSubscriptions, this._providerName); this._adapterCache = new SimpleQueueAdapterCache(this.options.CacheSize, this._providerName, loggerFactory); if (StreamFailureHandlerFactory == null) { StreamFailureHandlerFactory = qid => Task.FromResult <IStreamFailureHandler>(new NoOpStreamDeliveryFailureHandler()); } }
private void InitializeInternal(IQueueAdapter qAdapter, IQueueAdapterCache queueAdapterCache, IStreamFailureHandler failureHandler) { logger.Info(ErrorCode.PersistentStreamPullingAgent_02, "Init of {0} {1} on silo {2} for queue {3}.", GetType().Name, ((ISystemTargetBase)this).GrainId.ToDetailedString(), Silo, QueueId.ToStringWithHashCode()); // Remove cast once we cleanup queueAdapter = qAdapter; streamFailureHandler = failureHandler; lastTimeCleanedPubSubCache = DateTime.UtcNow; try { receiver = queueAdapter.CreateReceiver(QueueId); } catch (Exception exc) { logger.Error(ErrorCode.PersistentStreamPullingAgent_02, "Exception while calling IQueueAdapter.CreateNewReceiver.", exc); throw; } try { if (queueAdapterCache != null) { queueCache = queueAdapterCache.CreateQueueCache(QueueId); } } catch (Exception exc) { logger.Error(ErrorCode.PersistentStreamPullingAgent_23, "Exception while calling IQueueAdapterCache.CreateQueueCache.", exc); throw; } try { receiverInitTask = OrleansTaskExtentions.SafeExecute(() => receiver.Initialize(config.InitQueueTimeout)) .LogException(logger, ErrorCode.PersistentStreamPullingAgent_03, $"QueueAdapterReceiver {QueueId.ToStringWithHashCode()} failed to Initialize."); receiverInitTask.Ignore(); } catch { // Just ignore this exception and proceed as if Initialize has succeeded. // We already logged individual exceptions for individual calls to Initialize. No need to log again. } // Setup a reader for a new receiver. // Even if the receiver failed to initialise, treat it as OK and start pumping it. It's receiver responsibility to retry initialization. var randomTimerOffset = safeRandom.NextTimeSpan(config.GetQueueMsgsTimerPeriod); timer = RegisterTimer(AsyncTimerCallback, QueueId, randomTimerOffset, config.GetQueueMsgsTimerPeriod); IntValueStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_PUBSUB_CACHE_SIZE, StatisticUniquePostfix), () => pubSubCache.Count); logger.Info((int)ErrorCode.PersistentStreamPullingAgent_04, "Taking queue {0} under my responsibility.", QueueId.ToStringWithHashCode()); }
/// <summary> Init the factory.</summary> public virtual void Init(IProviderConfiguration config, string providerName, Logger logger, IServiceProvider serviceProvider) { if (config == null) { throw new ArgumentNullException("config"); } if (!config.Properties.TryGetValue(DataConnectionStringPropertyName, out dataConnectionString)) { throw new ArgumentException(String.Format("{0} property not set", DataConnectionStringPropertyName)); } if (!config.Properties.TryGetValue(DeploymentIdPropertyName, out deploymentId)) { throw new ArgumentException(String.Format("{0} property not set", DeploymentIdPropertyName)); } string messageVisibilityTimeoutRaw; if (config.Properties.TryGetValue(MessageVisibilityTimeoutPropertyName, out messageVisibilityTimeoutRaw)) { TimeSpan messageVisibilityTimeoutTemp; if (!TimeSpan.TryParse(messageVisibilityTimeoutRaw, out messageVisibilityTimeoutTemp)) { throw new ArgumentException(String.Format("Failed to parse {0} value '{1}' as a TimeSpan", MessageVisibilityTimeoutPropertyName, messageVisibilityTimeoutRaw)); } messageVisibilityTimeout = messageVisibilityTimeoutTemp; } else { messageVisibilityTimeout = null; } cacheSize = SimpleQueueAdapterCache.ParseSize(config, CacheSizeDefaultValue); string numQueuesString; numQueues = NumQueuesDefaultValue; if (config.Properties.TryGetValue(NumQueuesPropertyName, out numQueuesString)) { if (!int.TryParse(numQueuesString, out numQueues)) { throw new ArgumentException(String.Format("{0} invalid. Must be int", NumQueuesPropertyName)); } } this.providerName = providerName; streamQueueMapper = new HashRingBasedStreamQueueMapper(numQueues, providerName); adapterCache = new SimpleQueueAdapterCache(cacheSize, logger); if (StreamFailureHandlerFactory == null) { StreamFailureHandlerFactory = qid => Task.FromResult <IStreamFailureHandler>(new NoOpStreamDeliveryFailureHandler(false)); } }
public SimpleQueueAdapterCacheCursor(IQueueAdapterCache cache, Guid streamGuid, string streamNamespace, EventSequenceToken sequenceToken) { if (cache == null) { throw new ArgumentNullException("cache"); } this.cache = cache; this.streamGuid = streamGuid; this.streamNamespace = streamNamespace; cursor = this.cache.GetCursor(sequenceToken); }
public void Init(IProviderConfiguration config, string providerName, IServiceProvider serviceProvider) { Name = providerName; _publishSubscribe = serviceProvider.GetRequiredService <IPublishSubscribe>(); _serializationManager = serviceProvider.GetRequiredService <SerializationManager>(); ILoggerFactory loggerFactory = serviceProvider.GetRequiredService <ILoggerFactory>(); PublishSubscribeStreamConfiguration publishSubscribeStreamConfiguration = new PublishSubscribeStreamConfiguration(config); _adapterCache = new SimpleQueueAdapterCache(publishSubscribeStreamConfiguration.InMemoryCacheSize, providerName, loggerFactory); _streamQueueMapper = new HashRingBasedStreamQueueMapper(publishSubscribeStreamConfiguration.NumberOfQueues, providerName); }
private AzureQueueAdapterReceiver(QueueId queueId, AzureQueueDataManager queue, int cacheSize) { if (queueId == null) { throw new ArgumentNullException("queueId"); } if (queue == null) { throw new ArgumentNullException("queue"); } Id = queueId; this.queue = queue; cache = new SimpleQueueAdapterCache(cacheSize); }
internal PersistentStreamPullingManager( SystemTargetGrainId managerId, string strProviderName, IStreamProviderRuntime runtime, IStreamPubSub streamPubSub, IQueueAdapterFactory adapterFactory, IStreamQueueBalancer streamQueueBalancer, IStreamFilter streamFilter, StreamPullingAgentOptions options, ILoggerFactory loggerFactory, SiloAddress siloAddress) : base(managerId, siloAddress, lowPriority: false, loggerFactory) { if (string.IsNullOrWhiteSpace(strProviderName)) { throw new ArgumentNullException("strProviderName"); } if (runtime == null) { throw new ArgumentNullException("runtime", "IStreamProviderRuntime runtime reference should not be null"); } if (streamPubSub == null) { throw new ArgumentNullException("streamPubSub", "StreamPubSub reference should not be null"); } if (streamQueueBalancer == null) { throw new ArgumentNullException("streamQueueBalancer", "IStreamQueueBalancer streamQueueBalancer reference should not be null"); } queuesToAgentsMap = new Dictionary <QueueId, PersistentStreamPullingAgent>(); streamProviderName = strProviderName; providerRuntime = runtime; pubSub = streamPubSub; this.options = options; nonReentrancyGuarantor = new AsyncSerialExecutor(); latestRingNotificationSequenceNumber = 0; latestCommandNumber = 0; queueBalancer = streamQueueBalancer; this.streamFilter = streamFilter; this.adapterFactory = adapterFactory; queueAdapterCache = adapterFactory.GetQueueAdapterCache(); logger = loggerFactory.CreateLogger($"{GetType().FullName}.{streamProviderName}"); Log(ErrorCode.PersistentStreamPullingManager_01, "Created {0} for Stream Provider {1}.", GetType().Name, streamProviderName); this.loggerFactory = loggerFactory; IntValueStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_PULLING_AGENTS, strProviderName), () => queuesToAgentsMap.Count); }
internal PersistentStreamPullingManager( GrainId id, string strProviderName, IStreamProviderRuntime runtime, IStreamPubSub streamPubSub, IQueueAdapterFactory adapterFactory, IStreamQueueBalancer streamQueueBalancer, TimeSpan queueGetPeriod, TimeSpan initQueueTimeout, TimeSpan maxEvenDeliveryTime) : base(id, runtime.ExecutingSiloAddress) { if (string.IsNullOrWhiteSpace(strProviderName)) { throw new ArgumentNullException("strProviderName"); } if (runtime == null) { throw new ArgumentNullException("runtime", "IStreamProviderRuntime runtime reference should not be null"); } if (streamPubSub == null) { throw new ArgumentNullException("streamPubSub", "StreamPubSub reference should not be null"); } if (streamQueueBalancer == null) { throw new ArgumentNullException("streamQueueBalancer", "IStreamQueueBalancer streamQueueBalancer reference should not be null"); } queuesToAgentsMap = new Dictionary <QueueId, PersistentStreamPullingAgent>(); streamProviderName = strProviderName; providerRuntime = runtime; pubSub = streamPubSub; this.queueGetPeriod = queueGetPeriod; this.initQueueTimeout = initQueueTimeout; this.maxEvenDeliveryTime = maxEvenDeliveryTime; nonReentrancyGuarantor = new AsyncSerialExecutor(); latestRingNotificationSequenceNumber = 0; latestCommandNumber = 0; queueBalancer = streamQueueBalancer; this.adapterFactory = adapterFactory; queueAdapterCache = adapterFactory.GetQueueAdapterCache(); logger = providerRuntime.GetLogger(GetType().Name + "-" + streamProviderName); logger.Info((int)ErrorCode.PersistentStreamPullingManager_01, "Created {0} for Stream Provider {1}.", GetType().Name, streamProviderName); IntValueStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_PULLING_AGENTS, strProviderName), () => queuesToAgentsMap.Count); }
public AzureQueueAdapterFactory( string name, AzureQueueOptions options, SimpleQueueCacheOptions cacheOptions, IQueueDataAdapter <string, IBatchContainer> dataAdapter, IOptions <ClusterOptions> clusterOptions, ILoggerFactory loggerFactory) { this.providerName = name; this.options = options ?? throw new ArgumentNullException(nameof(options)); this.dataAdapter = dataAdapter ?? throw new ArgumentNullException(nameof(dataAdapter));; this.clusterOptions = clusterOptions.Value; this.loggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory)); this.streamQueueMapper = new AzureStreamQueueMapper(options.QueueNames, providerName); this.adapterCache = new SimpleQueueAdapterCache(cacheOptions, this.providerName, this.loggerFactory); }
public ServiceBusAdapterFactory( ILoggerFactory loggerFactory, string name, ServiceBusOptions options, SimpleQueueCacheOptions cacheOptions, IOptions <ClusterOptions> clusterOptions, SerializationManager serializationManager) { _providerName = name; _options = options ?? throw new ArgumentNullException(nameof(options)); _clusterOptions = clusterOptions.Value; SerializationManager = serializationManager ?? throw new ArgumentNullException(nameof(serializationManager)); _loggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory)); _streamQueueMapper = new ServiceBusStreamQueueMapper(options, $"{options.QueuePrefix}-{name}"); _adapterCache = new SimpleQueueAdapterCache(cacheOptions, _providerName, _loggerFactory); }
public SQSAdapterFactory( string name, SqsOptions sqsOptions, HashRingStreamQueueMapperOptions queueMapperOptions, SimpleQueueCacheOptions cacheOptions, IOptions <ClusterOptions> clusterOptions, Orleans.Serialization.Serializer serializer, ILoggerFactory loggerFactory) { this.providerName = name; this.sqsOptions = sqsOptions; this.clusterOptions = clusterOptions.Value; this.serializer = serializer.GetSerializer <SQSBatchContainer>(); this.loggerFactory = loggerFactory; streamQueueMapper = new HashRingBasedStreamQueueMapper(queueMapperOptions, this.providerName); adapterCache = new SimpleQueueAdapterCache(cacheOptions, this.providerName, this.loggerFactory); }
public PubSubAdapterFactory( string name, PubSubOptions options, HashRingStreamQueueMapperOptions queueMapperOptions, SimpleQueueCacheOptions cacheOptions, IServiceProvider serviceProvider, IOptions <ClusterOptions> clusterOptions, ILoggerFactory loggerFactory) { this._providerName = name; this.options = options; this.clusterOptions = clusterOptions.Value; this.loggerFactory = loggerFactory; this._adaptorFactory = () => ActivatorUtilities.GetServiceOrCreateInstance <TDataAdapter>(serviceProvider); this._streamQueueMapper = new HashRingBasedStreamQueueMapper(queueMapperOptions, this._providerName); this._adapterCache = new SimpleQueueAdapterCache(cacheOptions, this._providerName, loggerFactory); }
internal PersistentStreamPullingManager( GrainId id, string strProviderName, IStreamProviderRuntime runtime, IStreamPubSub streamPubSub, IQueueAdapterFactory adapterFactory, IStreamQueueBalancer streamQueueBalancer, PersistentStreamProviderConfig config) : base(id, runtime.ExecutingSiloAddress) { if (string.IsNullOrWhiteSpace(strProviderName)) { throw new ArgumentNullException("strProviderName"); } if (runtime == null) { throw new ArgumentNullException("runtime", "IStreamProviderRuntime runtime reference should not be null"); } if (streamPubSub == null) { throw new ArgumentNullException("streamPubSub", "StreamPubSub reference should not be null"); } if (streamQueueBalancer == null) { throw new ArgumentNullException("streamQueueBalancer", "IStreamQueueBalancer streamQueueBalancer reference should not be null"); } queuesToAgentsMap = new Dictionary<QueueId, PersistentStreamPullingAgent>(); streamProviderName = strProviderName; providerRuntime = runtime; pubSub = streamPubSub; this.config = config; nonReentrancyGuarantor = new AsyncSerialExecutor(); latestRingNotificationSequenceNumber = 0; latestCommandNumber = 0; queueBalancer = streamQueueBalancer; this.adapterFactory = adapterFactory; queueAdapterCache = adapterFactory.GetQueueAdapterCache(); logger = TraceLogger.GetLogger(GetType().Name + "-" + streamProviderName, TraceLogger.LoggerType.Provider); Log(ErrorCode.PersistentStreamPullingManager_01, "Created {0} for Stream Provider {1}.", GetType().Name, streamProviderName); IntValueStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_PULLING_AGENTS, strProviderName), () => queuesToAgentsMap.Count); queuePrintTimer = base.RegisterTimer(AsyncTimerCallback, null, QUEUES_PRINT_PERIOD, QUEUES_PRINT_PERIOD); }
internal PersistentStreamPullingManager( GrainId id, string strProviderName, IStreamProviderRuntime runtime, IStreamPubSub streamPubSub, IQueueAdapterFactory adapterFactory, IStreamQueueBalancer streamQueueBalancer, PersistentStreamProviderConfig config) : base(id, runtime.ExecutingSiloAddress) { if (string.IsNullOrWhiteSpace(strProviderName)) { throw new ArgumentNullException("strProviderName"); } if (runtime == null) { throw new ArgumentNullException("runtime", "IStreamProviderRuntime runtime reference should not be null"); } if (streamPubSub == null) { throw new ArgumentNullException("streamPubSub", "StreamPubSub reference should not be null"); } if (streamQueueBalancer == null) { throw new ArgumentNullException("streamQueueBalancer", "IStreamQueueBalancer streamQueueBalancer reference should not be null"); } queuesToAgentsMap = new Dictionary <QueueId, PersistentStreamPullingAgent>(); streamProviderName = strProviderName; providerRuntime = runtime; pubSub = streamPubSub; this.config = config; nonReentrancyGuarantor = new AsyncSerialExecutor(); latestRingNotificationSequenceNumber = 0; latestCommandNumber = 0; queueBalancer = streamQueueBalancer; this.adapterFactory = adapterFactory; queueAdapterCache = adapterFactory.GetQueueAdapterCache(); logger = LogManager.GetLogger(GetType().Name + "-" + streamProviderName, LoggerType.Provider); Log(ErrorCode.PersistentStreamPullingManager_01, "Created {0} for Stream Provider {1}.", GetType().Name, streamProviderName); IntValueStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_PULLING_AGENTS, strProviderName), () => queuesToAgentsMap.Count); queuePrintTimer = base.RegisterTimer(AsyncTimerCallback, null, QUEUES_PRINT_PERIOD, QUEUES_PRINT_PERIOD); }
public RabbitMqAdapterFactory( string providerName, IOptionsMonitor <RabbitMqOptions> rmqOptionsAccessor, IOptionsMonitor <SimpleQueueCacheOptions> cachingOptionsAccessor, IServiceProvider serviceProvider, ILoggerFactory loggerFactory, IRabbitMqStreamQueueMapperFactory streamQueueMapperFactory) { if (string.IsNullOrEmpty(providerName)) { throw new ArgumentNullException(nameof(providerName)); } if (rmqOptionsAccessor == null) { throw new ArgumentNullException(nameof(rmqOptionsAccessor)); } if (cachingOptionsAccessor == null) { throw new ArgumentNullException(nameof(cachingOptionsAccessor)); } if (serviceProvider == null) { throw new ArgumentNullException(nameof(serviceProvider)); } if (loggerFactory == null) { throw new ArgumentNullException(nameof(loggerFactory)); } if (streamQueueMapperFactory == null) { throw new ArgumentNullException(nameof(streamQueueMapperFactory)); } var rmqOptions = rmqOptionsAccessor.Get(providerName); var cachingOptions = cachingOptionsAccessor.Get(providerName); _cache = new SimpleQueueAdapterCache(cachingOptions, providerName, loggerFactory); _mapper = streamQueueMapperFactory.Get(providerName); _failureHandler = Task.FromResult <IStreamFailureHandler>(new NoOpStreamDeliveryFailureHandler(false)); var dataAdapter = serviceProvider.GetServiceByName <IQueueDataAdapter <RabbitMqMessage, IEnumerable <IBatchContainer> > >(providerName) ?? RabbitMqDataAdapter.Create(serviceProvider, providerName); _adapter = new RabbitMqAdapter(rmqOptions, dataAdapter, providerName, _mapper, serviceProvider.GetRequiredServiceByName <IRabbitMqConnectorFactory>(providerName)); }
public AzureQueueAdapterFactory( string name, AzureQueueOptions options, SimpleQueueCacheOptions cacheOptions, IServiceProvider serviceProvider, IOptions <ClusterOptions> clusterOptions, SerializationManager serializationManager, ILoggerFactory loggerFactory) { this.providerName = name; this.options = options ?? throw new ArgumentNullException(nameof(options)); this.clusterOptions = clusterOptions.Value; this.SerializationManager = serializationManager ?? throw new ArgumentNullException(nameof(serializationManager)); this.loggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory)); this.dataAadaptorFactory = () => ActivatorUtilities.GetServiceOrCreateInstance <TDataAdapter>(serviceProvider); this.streamQueueMapper = new AzureStreamQueueMapper(options.QueueNames, providerName); this.adapterCache = new SimpleQueueAdapterCache(cacheOptions, this.providerName, this.loggerFactory); }
public TestAdapterFactory( string name, SimpleQueueCacheOptions cacheOptions, Func <IEnumerable <IBatchContainer> > queueMessagesProvider, Action <IEnumerable <IBatchContainer> > onMessagesDelivered, ILoggerFactory loggerFactory ) { _providerName = name; _queueMessagesProvider = queueMessagesProvider ?? throw new ArgumentNullException(nameof(queueMessagesProvider)); _onMessagesDelivered = onMessagesDelivered ?? throw new ArgumentNullException(nameof(onMessagesDelivered)); _loggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory)); _streamQueueMapper = new HashRingBasedStreamQueueMapper(new HashRingStreamQueueMapperOptions() { TotalQueueCount = 1 }, _providerName); _adapterCache = new SimpleQueueAdapterCache(cacheOptions, _providerName, _loggerFactory); }
public SQSAdapterFactory( string name, SqsOptions sqsOptions, HashRingStreamQueueMapperOptions queueMapperOptions, SimpleQueueCacheOptions cacheOptions, IServiceProvider serviceProvider, IOptions <ClusterOptions> clusterOptions, SerializationManager serializationManager, ILoggerFactory loggerFactory) { this.providerName = name; this.sqsOptions = sqsOptions; this.clusterOptions = clusterOptions.Value; this.serializationManager = serializationManager; this.loggerFactory = loggerFactory; streamQueueMapper = new HashRingBasedStreamQueueMapper(queueMapperOptions, this.providerName); adapterCache = new SimpleQueueAdapterCache(cacheOptions, this.providerName, this.loggerFactory); }
public RabbitMQAdapterFactory( string name, RabbitMQStreamProviderOptions options, HashRingStreamQueueMapperOptions queueMapperOptions, SimpleQueueCacheOptions cacheOptions, IServiceProvider serviceProvider, IOptions <ClusterOptions> clusterOptions, SerializationManager serializationManager, ILoggerFactory loggerFactory) { providerName = name; this.options = options ?? throw new ArgumentNullException(nameof(options)); this.clusterOptions = clusterOptions.Value; //this.SerializationManager = serializationManager ?? throw new ArgumentNullException(nameof(serializationManager)); this.loggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory)); streamQueueMapper = new HashRingBasedStreamQueueMapper(queueMapperOptions, providerName); adapterCache = new SimpleQueueAdapterCache(cacheOptions, providerName, this.loggerFactory); mapper = ActivatorUtilities.GetServiceOrCreateInstance <TMapper>(serviceProvider); }
internal PersistentStreamPullingAgent( SystemTargetGrainId id, string strProviderName, ILoggerFactory loggerFactory, IStreamPubSub streamPubSub, IStreamFilter streamFilter, QueueId queueId, StreamPullingAgentOptions options, SiloAddress siloAddress, IQueueAdapter queueAdapter, IQueueAdapterCache queueAdapterCache, IStreamFailureHandler streamFailureHandler) : base(id, siloAddress, true, loggerFactory) { if (strProviderName == null) { throw new ArgumentNullException("runtime", "PersistentStreamPullingAgent: strProviderName should not be null"); } QueueId = queueId; streamProviderName = strProviderName; pubSub = streamPubSub; this.streamFilter = streamFilter; pubSubCache = new Dictionary <InternalStreamId, StreamConsumerCollection>(); this.options = options; this.queueAdapter = queueAdapter ?? throw new ArgumentNullException(nameof(queueAdapter)); this.streamFailureHandler = streamFailureHandler ?? throw new ArgumentNullException(nameof(streamFailureHandler));; this.queueAdapterCache = queueAdapterCache; numMessages = 0; logger = loggerFactory.CreateLogger($"{this.GetType().Namespace}.{streamProviderName}"); logger.LogInformation( (int)ErrorCode.PersistentStreamPullingAgent_01, "Created {Name} {Id} for Stream Provider {StreamProvider} on silo {Silo} for Queue {Queue}.", GetType().Name, ((ISystemTargetBase)this).GrainId.ToString(), streamProviderName, Silo, QueueId.ToStringWithHashCode()); numReadMessagesCounter = CounterStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_READ_MESSAGES, StatisticUniquePostfix)); numSentMessagesCounter = CounterStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_SENT_MESSAGES, StatisticUniquePostfix)); }
public RabbitMqAdapterFactory( string providerName, RabbitMqOptions rmqOptions, CachingOptions cachingOptions, IServiceProvider serviceProvider, ILoggerFactory loggerFactory) { if (string.IsNullOrEmpty(providerName)) { throw new ArgumentNullException(nameof(providerName)); } if (rmqOptions == null) { throw new ArgumentNullException(nameof(rmqOptions)); } if (cachingOptions == null) { throw new ArgumentNullException(nameof(cachingOptions)); } if (serviceProvider == null) { throw new ArgumentNullException(nameof(serviceProvider)); } if (loggerFactory == null) { throw new ArgumentNullException(nameof(loggerFactory)); } _cache = new SimpleQueueAdapterCache(new SimpleQueueCacheOptions { CacheSize = cachingOptions.CacheSize }, providerName, loggerFactory); _mapper = new HashRingBasedStreamQueueMapper(new HashRingStreamQueueMapperOptions { TotalQueueCount = rmqOptions.NumberOfQueues }, rmqOptions.QueueNamePrefix); _failureHandler = Task.FromResult <IStreamFailureHandler>(new NoOpStreamDeliveryFailureHandler(false)); var serializer = typeof(TSerializer) == typeof(DefaultBatchContainerSerializer) ? new DefaultBatchContainerSerializer(serviceProvider.GetRequiredService <SerializationManager>()) : (IBatchContainerSerializer) new TSerializer(); _adapter = new RabbitMqAdapter(rmqOptions, cachingOptions, serializer, _mapper, providerName, loggerFactory); }
public void Init(IProviderConfiguration config, string providerName, Logger logger, IServiceProvider serviceProvider) { if (config == null) { throw new ArgumentNullException(nameof(config)); } if (logger == null) { throw new ArgumentNullException(nameof(logger)); } if (string.IsNullOrEmpty(providerName)) { throw new ArgumentNullException(nameof(providerName)); } _options = new RabbitMqStreamProviderOptions(config); _providerName = providerName; _cache = new ConcurrentQueueAdapterCache(_options.CacheSize); _mapper = new HashRingBasedStreamQueueMapper(_options.NumberOfQueues, _options.QueueNamePrefix); _failureHandler = Task.FromResult <IStreamFailureHandler>(new NoOpStreamDeliveryFailureHandler(false)); _adapter = new RabbitMqAdapter(_options, serviceProvider.GetRequiredService <SerializationManager>(), _mapper, _providerName, logger); }
internal PersistentStreamPullingManager( GrainId id, string strProviderName, IStreamProviderRuntime runtime, IQueueAdapterFactory adapterFactory, IStreamQueueBalancer streamQueueBalancer, TimeSpan queueGetPeriod, TimeSpan initQueueTimeout) : base(id, runtime.ExecutingSiloAddress) { if (string.IsNullOrWhiteSpace(strProviderName)) { throw new ArgumentNullException("strProviderName"); } if (runtime == null) { throw new ArgumentNullException("runtime", "IStreamProviderRuntime runtime reference should not be null"); } if (streamQueueBalancer == null) { throw new ArgumentNullException("streamQueueBalancer", "IStreamQueueBalancer streamQueueBalancer reference should not be null"); } queuesToAgentsMap = new Dictionary<QueueId, PersistentStreamPullingAgent>(); streamProviderName = strProviderName; providerRuntime = runtime; this.queueGetPeriod = queueGetPeriod; this.initQueueTimeout = initQueueTimeout; nonReentrancyGuarantor = new AsyncSerialExecutor(); latestRingNotificationSequenceNumber = 0; queueBalancer = streamQueueBalancer; queueAdapterCache = adapterFactory.GetQueueAdapterCache(); logger = providerRuntime.GetLogger(GetType().Name + "-" + streamProviderName); logger.Info((int)ErrorCode.PersistentStreamPullingManager_01, "Created {0} for Stream Provider {1}.", GetType().Name, streamProviderName); IntValueStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_PULLING_AGENTS, strProviderName), () => queuesToAgentsMap.Count); }
public KafkaQueueAdapterFactory( string name, KafkaStreamProviderOptions options, ILoggerFactory loggerFactory) { if (string.IsNullOrEmpty(name)) { throw new ArgumentNullException(nameof(name)); } _options = options ?? throw new ArgumentNullException(nameof(options)); _loggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory)); _logger = _loggerFactory.CreateLogger <KafkaQueueAdapterFactory>(); _providerName = name; _streamQueueMapper = new HashRingBasedStreamQueueMapper( new HashRingStreamQueueMapperOptions { TotalQueueCount = _options.PartitionsCount }, name); _adapterCache = new SimpleQueueAdapterCache(new SimpleQueueCacheOptions { CacheSize = options.Cache.CacheSize }, _providerName, _loggerFactory); }
public IQueueAdapterCache GetQueueAdapterCache() { return _adapterCache ?? (_adapterCache = new Cache.QueueAdapterCache(this, TimeSpan.FromSeconds(_timeLimit), _logger)); }
private void InitializeInternal(IQueueAdapter qAdapter, IQueueAdapterCache queueAdapterCache, IStreamFailureHandler failureHandler) { logger.Info(ErrorCode.PersistentStreamPullingAgent_02, "Init of {0} {1} on silo {2} for queue {3}.", GetType().Name, GrainId.ToDetailedString(), Silo, QueueId.ToStringWithHashCode()); // Remove cast once we cleanup queueAdapter = qAdapter; streamFailureHandler = failureHandler; lastTimeCleanedPubSubCache = DateTime.UtcNow; try { receiver = queueAdapter.CreateReceiver(QueueId); } catch (Exception exc) { logger.Error(ErrorCode.PersistentStreamPullingAgent_02, "Exception while calling IQueueAdapter.CreateNewReceiver.", exc); throw; } try { if (queueAdapterCache != null) { queueCache = queueAdapterCache.CreateQueueCache(QueueId); } } catch (Exception exc) { logger.Error(ErrorCode.PersistentStreamPullingAgent_23, "Exception while calling IQueueAdapterCache.CreateQueueCache.", exc); throw; } try { receiverInitTask = OrleansTaskExtentions.SafeExecute(() => receiver.Initialize(config.InitQueueTimeout)) .LogException(logger, ErrorCode.PersistentStreamPullingAgent_03, $"QueueAdapterReceiver {QueueId.ToStringWithHashCode()} failed to Initialize."); receiverInitTask.Ignore(); } catch { // Just ignore this exception and proceed as if Initialize has succeeded. // We already logged individual exceptions for individual calls to Initialize. No need to log again. } // Setup a reader for a new receiver. // Even if the receiver failed to initialise, treat it as OK and start pumping it. It's receiver responsibility to retry initialization. var randomTimerOffset = safeRandom.NextTimeSpan(config.GetQueueMsgsTimerPeriod); timer = RegisterTimer(AsyncTimerCallback, QueueId, randomTimerOffset, config.GetQueueMsgsTimerPeriod); logger.Info((int)ErrorCode.PersistentStreamPullingAgent_04, "Taking queue {0} under my responsibility.", QueueId.ToStringWithHashCode()); }
public IQueueAdapterCache GetQueueAdapterCache() { return _adapterCache ?? (_adapterCache = new MemoryCache.MySimpleQueueAdapterCache(this, _logger)); }
public void Init(IProviderConfiguration config, string providerName, Logger logger, IServiceProvider serviceProvider) { if (config == null) throw new ArgumentNullException(nameof(config)); if (!config.Properties.TryGetValue(RABBITMQ_CONNECTION_STRING, out _dataConnectionString)) throw new ArgumentException($"{RABBITMQ_CONNECTION_STRING} property not set"); if (!config.Properties.TryGetValue(DEPLOYMENT_ID, out _deploymentId)) throw new ArgumentException($"{DEPLOYMENT_ID} property not set"); string cacheSizeString; _cacheSize = DEFAULT_CACHE_SIZE; if (config.Properties.TryGetValue(CACHE_SIZE_PARAM, out cacheSizeString)) if (!int.TryParse(cacheSizeString, out _cacheSize)) throw new ArgumentException($"{CACHE_SIZE_PARAM} invalid. Must be int"); string numQueuesString; _numQueues = DEFAULT_NUM_QUEUES; if (config.Properties.TryGetValue(NUM_QUEUES_PARAM, out numQueuesString)) if (!int.TryParse(numQueuesString, out _numQueues)) throw new ArgumentException($"{NUM_QUEUES_PARAM} invalid. Must be int"); _providerName = providerName; _streamQueueMapper = new HashRingBasedStreamQueueMapper(_numQueues, providerName); _adapterCache = new SimpleQueueAdapterCache(_cacheSize, logger); }