Пример #1
0
        /// <summary>
        /// Create a IEventHubQueueCacheFactory. It will create a EventHubQueueCacheFactory by default.
        /// User can override this function to return their own implementation of IEventHubQueueCacheFactory,
        /// and other customization of IEventHubQueueCacheFactory if they may.
        /// </summary>
        /// <param name="providerSettings"></param>
        /// <returns></returns>
        protected virtual IEventHubQueueCacheFactory CreateCacheFactory(EventHubStreamProviderSettings providerSettings)
        {
            var eventHubPath     = this.hubSettings.Path;
            var sharedDimensions = new EventHubMonitorAggregationDimensions(eventHubPath);

            return(new EventHubQueueCacheFactory(providerSettings, this.SerializationManager, sharedDimensions, this.loggerFactory));
        }
Пример #2
0
        /// <summary>
        /// Function used to configure cache pressure monitors for EventHubQueueCache.
        /// User can override this function to provide more customization on cache pressure monitors
        /// </summary>
        /// <param name="cache"></param>
        /// <param name="providerSettings"></param>
        /// <param name="cacheLogger"></param>
        protected virtual void AddCachePressureMonitors(IEventHubQueueCache cache, EventHubStreamProviderSettings providerSettings,
                                                        Logger cacheLogger)
        {
            if (providerSettings.AveragingCachePressureMonitorFlowControlThreshold.HasValue)
            {
                var avgMonitor = new AveragingCachePressureMonitor(
                    providerSettings.AveragingCachePressureMonitorFlowControlThreshold.Value, cacheLogger);
                cache.AddCachePressureMonitor(avgMonitor);
            }

            if (providerSettings.SlowConsumingMonitorPressureWindowSize.HasValue ||
                providerSettings.SlowConsumingMonitorFlowControlThreshold.HasValue)
            {
                var slowConsumeMonitor = new SlowConsumingPressureMonitor(cacheLogger);
                if (providerSettings.SlowConsumingMonitorFlowControlThreshold.HasValue)
                {
                    slowConsumeMonitor.FlowControlThreshold = providerSettings.SlowConsumingMonitorFlowControlThreshold.Value;
                }
                if (providerSettings.SlowConsumingMonitorPressureWindowSize.HasValue)
                {
                    slowConsumeMonitor.PressureWindowSize = providerSettings.SlowConsumingMonitorPressureWindowSize.Value;
                }

                cache.AddCachePressureMonitor(slowConsumeMonitor);
            }
        }
 /// <summary>
 /// Constructor for EventHubQueueCacheFactory
 /// </summary>
 /// <param name="providerSettings"></param>
 /// <param name="serializationManager"></param>
 public EventHubQueueCacheFactory(EventHubStreamProviderSettings providerSettings,
                                  SerializationManager serializationManager
                                  )
 {
     _providerSettings     = providerSettings;
     _serializationManager = serializationManager;
     _timePurge            = new TimePurgePredicate(_providerSettings.DataMinTimeInCache, _providerSettings.DataMaxAgeInCache);
 }
Пример #4
0
        /// <summary>
        /// Default function to be called to create an EventhubQueueCache in IEventHubQueueCacheFactory.CreateCache method. User can
        /// override this method to add more customization.
        /// </summary>
        protected virtual IEventHubQueueCache CreateCache(string partition, EventHubStreamProviderSettings providerSettings, IStreamQueueCheckpointer <string> checkpointer,
                                                          Logger cacheLogger, IObjectPool <FixedSizeBuffer> bufferPool, string blockPoolId, TimePurgePredicate timePurge,
                                                          SerializationManager serializationManager, EventHubMonitorAggregationDimensions sharedDimensions, ITelemetryProducer telemetryProducer)
        {
            var cacheMonitorDimensions = new EventHubCacheMonitorDimensions(sharedDimensions, partition, blockPoolId);
            var cacheMonitor           = this.CacheMonitorFactory(cacheMonitorDimensions, cacheLogger, telemetryProducer);

            return(new EventHubQueueCache(checkpointer, bufferPool, timePurge, cacheLogger, serializationManager, cacheMonitor, providerSettings.StatisticMonitorWriteInterval));
        }
Пример #5
0
        /// <summary>
        /// Create a IEventHubQueueCacheFactory. It will create a EventHubQueueCacheFactory by default.
        /// User can override this function to return their own implementation of IEventHubQueueCacheFactory,
        /// and other customization of IEventHubQueueCacheFactory if they may.
        /// </summary>
        /// <param name="providerSettings"></param>
        /// <returns></returns>
        protected virtual IEventHubQueueCacheFactory CreateCacheFactory(EventHubStreamProviderSettings providerSettings)
        {
            var globalConfig     = this.serviceProvider.GetService <GlobalConfiguration>();
            var nodeConfig       = this.serviceProvider.GetService <NodeConfiguration>();
            var eventHubPath     = hubSettings.Path;
            var sharedDimensions = new EventHubMonitorAggregationDimensions(globalConfig, nodeConfig, eventHubPath);

            return(new EventHubQueueCacheFactory(providerSettings, SerializationManager, sharedDimensions));
        }
Пример #6
0
        /// <summary>
        /// Factory initialization.
        /// Provider config must contain the event hub settings type or the settings themselves.
        /// EventHubSettingsType is recommended for consumers that do not want to include secure information in the cluster configuration.
        /// </summary>
        /// <param name="providerCfg"></param>
        /// <param name="providerName"></param>
        /// <param name="log"></param>
        /// <param name="svcProvider"></param>
        public virtual void Init(IProviderConfiguration providerCfg, string providerName, Logger log, IServiceProvider svcProvider)
        {
            if (providerCfg == null)
            {
                throw new ArgumentNullException(nameof(providerCfg));
            }
            if (string.IsNullOrWhiteSpace(providerName))
            {
                throw new ArgumentNullException(nameof(providerName));
            }
            if (log == null)
            {
                throw new ArgumentNullException(nameof(log));
            }

            providerConfig  = providerCfg;
            serviceProvider = svcProvider;
            receivers       = new ConcurrentDictionary <QueueId, EventHubAdapterReceiver>();

            adapterSettings = new EventHubStreamProviderSettings(providerName);
            adapterSettings.PopulateFromProviderConfig(providerConfig);
            hubSettings = adapterSettings.GetEventHubSettings(providerConfig, serviceProvider);
            client      = EventHubClient.CreateFromConnectionString(hubSettings.ConnectionString, hubSettings.Path);

            if (CheckpointerFactory == null)
            {
                checkpointerSettings = adapterSettings.GetCheckpointerSettings(providerConfig, serviceProvider);
                CheckpointerFactory  = partition => EventHubCheckpointer.Create(checkpointerSettings, adapterSettings.StreamProviderName, partition);
            }

            if (CacheFactory == null)
            {
                var bufferPool = new FixedSizeObjectPool <FixedSizeBuffer>(adapterSettings.CacheSizeMb, () => new FixedSizeBuffer(1 << 20));
                var timePurge  = new TimePurgePredicate(adapterSettings.DataMinTimeInCache, adapterSettings.DataMaxAgeInCache);
                CacheFactory = (partition, checkpointer, cacheLogger) => new EventHubQueueCache(checkpointer, bufferPool, timePurge, cacheLogger);
            }

            if (StreamFailureHandlerFactory == null)
            {
                //TODO: Add a queue specific default failure handler with reasonable error reporting.
                StreamFailureHandlerFactory = partition => Task.FromResult <IStreamFailureHandler>(new NoOpStreamDeliveryFailureHandler());
            }

            if (QueueMapperFactory == null)
            {
                QueueMapperFactory = partitions => new EventHubQueueMapper(partitionIds, adapterSettings.StreamProviderName);
            }

            if (ReceiverMonitorFactory == null)
            {
                ReceiverMonitorFactory = (hubPath, partition, receiverLogger) => new DefaultEventHubReceiverMonitor(hubPath, partition, receiverLogger.GetSubLogger("monitor", "-"));
            }

            logger = log.GetLogger($"EventHub.{hubSettings.Path}");
        }
Пример #7
0
        /// <summary>
        /// Factory initialization.
        /// Provider config must contain the event hub settings type or the settings themselves.
        /// EventHubSettingsType is recommended for consumers that do not want to include secure information in the cluster configuration.
        /// </summary>
        /// <param name="providerCfg"></param>
        /// <param name="providerName"></param>
        /// <param name="log"></param>
        /// <param name="svcProvider"></param>
        public virtual void Init(IProviderConfiguration providerCfg, string providerName, Logger log, IServiceProvider svcProvider)
        {
            if (providerCfg == null)
            {
                throw new ArgumentNullException(nameof(providerCfg));
            }
            if (string.IsNullOrWhiteSpace(providerName))
            {
                throw new ArgumentNullException(nameof(providerName));
            }
            if (log == null)
            {
                throw new ArgumentNullException(nameof(log));
            }

            providerConfig            = providerCfg;
            serviceProvider           = svcProvider;
            this.loggerFactory        = serviceProvider.GetRequiredService <ILoggerFactory>();
            receivers                 = new ConcurrentDictionary <QueueId, EventHubAdapterReceiver>();
            this.SerializationManager = this.serviceProvider.GetRequiredService <SerializationManager>();
            adapterSettings           = new EventHubStreamProviderSettings(providerName);
            adapterSettings.PopulateFromProviderConfig(providerConfig);
            hubSettings            = adapterSettings.GetEventHubSettings(providerConfig, serviceProvider);
            this.telemetryProducer = serviceProvider.GetService <ITelemetryProducer>();

            InitEventHubClient();
            if (CheckpointerFactory == null)
            {
                checkpointerSettings = adapterSettings.GetCheckpointerSettings(providerConfig, serviceProvider);
                CheckpointerFactory  = partition => EventHubCheckpointer.Create(checkpointerSettings, adapterSettings.StreamProviderName, partition, this.loggerFactory);
            }

            if (CacheFactory == null)
            {
                CacheFactory = CreateCacheFactory(adapterSettings).CreateCache;
            }

            if (StreamFailureHandlerFactory == null)
            {
                //TODO: Add a queue specific default failure handler with reasonable error reporting.
                StreamFailureHandlerFactory = partition => Task.FromResult <IStreamFailureHandler>(new NoOpStreamDeliveryFailureHandler());
            }

            if (QueueMapperFactory == null)
            {
                QueueMapperFactory = partitions => new EventHubQueueMapper(partitions, adapterSettings.StreamProviderName);
            }

            if (ReceiverMonitorFactory == null)
            {
                ReceiverMonitorFactory = (dimensions, logger, telemetryProducer) => new DefaultEventHubReceiverMonitor(dimensions, telemetryProducer);
            }

            logger = log.GetLogger($"EventHub.{hubSettings.Path}");
        }
Пример #8
0
 /// <summary>
 /// Constructor for EventHubQueueCacheFactory
 /// </summary>
 /// <param name="providerSettings"></param>
 /// <param name="serializationManager"></param>
 /// <param name="sharedDimensions">shared dimensions between cache monitor and block pool monitor</param>
 /// <param name="cacheMonitorFactory"></param>
 /// <param name="blockPoolMonitorFactory"></param>
 public EventHubQueueCacheFactory(EventHubStreamProviderSettings providerSettings,
                                  SerializationManager serializationManager, EventHubMonitorAggregationDimensions sharedDimensions,
                                  Func <EventHubCacheMonitorDimensions, Logger, ITelemetryProducer, ICacheMonitor> cacheMonitorFactory             = null,
                                  Func <EventHubBlockPoolMonitorDimensions, Logger, ITelemetryProducer, IBlockPoolMonitor> blockPoolMonitorFactory = null)
 {
     this.providerSettings        = providerSettings;
     this.serializationManager    = serializationManager;
     this.timePurge               = new TimePurgePredicate(this.providerSettings.DataMinTimeInCache, this.providerSettings.DataMaxAgeInCache);
     this.sharedDimensions        = sharedDimensions;
     this.CacheMonitorFactory     = cacheMonitorFactory == null?(dimensions, logger, telemetryProducer) => new DefaultEventHubCacheMonitor(dimensions, telemetryProducer) : cacheMonitorFactory;
     this.BlockPoolMonitorFactory = blockPoolMonitorFactory == null? (dimensions, logger, telemetryProducer) => new DefaultEventHubBlockPoolMonitor(dimensions, telemetryProducer) : blockPoolMonitorFactory;
 }
Пример #9
0
 /// <summary>
 /// Function used to configure BufferPool for EventHubQueueCache. User can override this function to provide more customization on BufferPool creation
 /// </summary>
 protected virtual IObjectPool <FixedSizeBuffer> CreateBufferPool(EventHubStreamProviderSettings providerSettings, Logger logger, EventHubMonitorAggregationDimensions sharedDimensions, ITelemetryProducer telemetryProducer, out string blockPoolId)
 {
     if (this.bufferPool == null)
     {
         var bufferSize = 1 << 20;
         this.bufferPoolId = $"BlockPool-{new Guid().ToString()}-BlockSize-{bufferSize}";
         var monitorDimensions = new EventHubBlockPoolMonitorDimensions(sharedDimensions, this.bufferPoolId);
         var objectPoolMonitor = new ObjectPoolMonitorBridge(this.BlockPoolMonitorFactory(monitorDimensions, logger, telemetryProducer), bufferSize);
         this.bufferPool = new ObjectPool <FixedSizeBuffer>(() => new FixedSizeBuffer(bufferSize),
                                                            objectPoolMonitor, providerSettings.StatisticMonitorWriteInterval);
     }
     blockPoolId = this.bufferPoolId;
     return(this.bufferPool);
 }
Пример #10
0
 /// <summary>
 /// Create a IEventHubQueueCacheFactory. It will create a EventHubQueueCacheFactory by default.
 /// User can override this function to return their own implementation of IEventHubQueueCacheFactory,
 /// and other customization of IEventHubQueueCacheFactory if they may.
 /// </summary>
 /// <param name="providerSettings"></param>
 /// <returns></returns>
 protected virtual IEventHubQueueCacheFactory CreateCacheFactory(EventHubStreamProviderSettings providerSettings)
 {
     return(new EventHubQueueCacheFactory(providerSettings, SerializationManager));
 }
Пример #11
0
        /// <summary>
        /// Factory initialization.
        /// Provider config must contain the event hub settings type or the settings themselves.
        /// EventHubSettingsType is recommended for consumers that do not want to include secure information in the cluster configuration.
        /// </summary>
        /// <param name="providerCfg"></param>
        /// <param name="providerName"></param>
        /// <param name="log"></param>
        /// <param name="svcProvider"></param>
        public virtual void Init(IProviderConfiguration providerCfg, string providerName, Logger log, IServiceProvider svcProvider)
        {
            if (providerCfg == null)
            {
                throw new ArgumentNullException(nameof(providerCfg));
            }
            if (string.IsNullOrWhiteSpace(providerName))
            {
                throw new ArgumentNullException(nameof(providerName));
            }
            if (log == null)
            {
                throw new ArgumentNullException(nameof(log));
            }

            providerConfig            = providerCfg;
            serviceProvider           = svcProvider;
            receivers                 = new ConcurrentDictionary <QueueId, EventHubAdapterReceiver>();
            this.SerializationManager = this.serviceProvider.GetRequiredService <SerializationManager>();
            adapterSettings           = new EventHubStreamProviderSettings(providerName);
            adapterSettings.PopulateFromProviderConfig(providerConfig);
            hubSettings = adapterSettings.GetEventHubSettings(providerConfig, serviceProvider);
#if NETSTANDARD
            var connectionStringBuilder = new EventHubsConnectionStringBuilder(hubSettings.ConnectionString)
            {
                EntityPath = hubSettings.Path
            };
            client = EventHubClient.CreateFromConnectionString(connectionStringBuilder.ToString());
#else
            client = EventHubClient.CreateFromConnectionString(hubSettings.ConnectionString, hubSettings.Path);
#endif

            if (CheckpointerFactory == null)
            {
                checkpointerSettings = adapterSettings.GetCheckpointerSettings(providerConfig, serviceProvider);
                CheckpointerFactory  = partition => EventHubCheckpointer.Create(checkpointerSettings, adapterSettings.StreamProviderName, partition);
            }

            if (CacheFactory == null)
            {
                CacheFactory = CreateCacheFactory(adapterSettings).CreateCache;
            }

            if (StreamFailureHandlerFactory == null)
            {
                //TODO: Add a queue specific default failure handler with reasonable error reporting.
                StreamFailureHandlerFactory = partition => Task.FromResult <IStreamFailureHandler>(new NoOpStreamDeliveryFailureHandler());
            }

            if (QueueMapperFactory == null)
            {
                QueueMapperFactory = partitions => new EventHubQueueMapper(partitionIds, adapterSettings.StreamProviderName);
            }

            if (ReceiverMonitorFactory == null)
            {
                ReceiverMonitorFactory = (hubPath, partition, receiverLogger) => new DefaultEventHubReceiverMonitor(hubPath, partition, receiverLogger.GetSubLogger("monitor", "-"));
            }

            logger = log.GetLogger($"EventHub.{hubSettings.Path}");
        }
 /// <summary>
 /// Function used to configure BufferPool for EventHubQueueCache. User can override this function to provide more customization on BufferPool creation
 /// </summary>
 /// <param name="providerSettings"></param>
 /// <returns></returns>
 protected virtual IObjectPool <FixedSizeBuffer> CreateBufferPool(EventHubStreamProviderSettings providerSettings)
 {
     return(_bufferPool ?? (_bufferPool = new FixedSizeObjectPool <FixedSizeBuffer>(providerSettings.CacheSizeMb,
                                                                                    () => new FixedSizeBuffer(1 << 20))));
 }