/// <summary>
 /// Constructor
 /// </summary>
 /// <param name="logger"></param>
 /// <param name="timePurage"></param>
 public EventHubCacheEvictionStrategy(Logger logger, TimePurgePredicate timePurage = null)
 {
     this.inUseBuffers  = new Queue <FixedSizeBuffer>();
     this.logger        = logger.GetSubLogger(this.GetType().Name);
     this.purgedBuffers = new Queue <FixedSizeBuffer>();
     this.timePurge     = timePurage ?? TimePurgePredicate.Default;
 }
 /// <summary>
 /// Constructor for EventHubQueueCacheFactory
 /// </summary>
 /// <param name="providerSettings"></param>
 /// <param name="serializationManager"></param>
 public EventHubQueueCacheFactory(EventHubStreamProviderSettings providerSettings,
                                  SerializationManager serializationManager
                                  )
 {
     _providerSettings     = providerSettings;
     _serializationManager = serializationManager;
     _timePurge            = new TimePurgePredicate(_providerSettings.DataMinTimeInCache, _providerSettings.DataMaxAgeInCache);
 }
 /// <summary>
 /// Cache data adapter that adapts EventHub's EventData to CachedEventHubMessage used in cache
 /// </summary>
 /// <param name="bufferPool"></param>
 /// <param name="timePurage"></param>
 public EventHubDataAdapter(IObjectPool <FixedSizeBuffer> bufferPool, TimePurgePredicate timePurage = null)
 {
     if (bufferPool == null)
     {
         throw new ArgumentNullException(nameof(bufferPool));
     }
     this.bufferPool = bufferPool;
     this.timePurage = timePurage ?? TimePurgePredicate.Default;
 }
Exemple #4
0
        /// <summary>
        /// Factory initialization.
        /// Provider config must contain the event hub settings type or the settings themselves.
        /// EventHubSettingsType is recommended for consumers that do not want to include secure information in the cluster configuration.
        /// </summary>
        /// <param name="providerCfg"></param>
        /// <param name="providerName"></param>
        /// <param name="log"></param>
        /// <param name="svcProvider"></param>
        public virtual void Init(IProviderConfiguration providerCfg, string providerName, Logger log, IServiceProvider svcProvider)
        {
            if (providerCfg == null)
            {
                throw new ArgumentNullException(nameof(providerCfg));
            }
            if (string.IsNullOrWhiteSpace(providerName))
            {
                throw new ArgumentNullException(nameof(providerName));
            }
            if (log == null)
            {
                throw new ArgumentNullException(nameof(log));
            }

            providerConfig  = providerCfg;
            serviceProvider = svcProvider;
            receivers       = new ConcurrentDictionary <QueueId, EventHubAdapterReceiver>();

            adapterSettings = new EventHubStreamProviderSettings(providerName);
            adapterSettings.PopulateFromProviderConfig(providerConfig);
            hubSettings = adapterSettings.GetEventHubSettings(providerConfig, serviceProvider);
            client      = EventHubClient.CreateFromConnectionString(hubSettings.ConnectionString, hubSettings.Path);

            if (CheckpointerFactory == null)
            {
                checkpointerSettings = adapterSettings.GetCheckpointerSettings(providerConfig, serviceProvider);
                CheckpointerFactory  = partition => EventHubCheckpointer.Create(checkpointerSettings, adapterSettings.StreamProviderName, partition);
            }

            if (CacheFactory == null)
            {
                var bufferPool = new FixedSizeObjectPool <FixedSizeBuffer>(adapterSettings.CacheSizeMb, () => new FixedSizeBuffer(1 << 20));
                var timePurge  = new TimePurgePredicate(adapterSettings.DataMinTimeInCache, adapterSettings.DataMaxAgeInCache);
                CacheFactory = (partition, checkpointer, cacheLogger) => new EventHubQueueCache(checkpointer, bufferPool, timePurge, cacheLogger);
            }

            if (StreamFailureHandlerFactory == null)
            {
                //TODO: Add a queue specific default failure handler with reasonable error reporting.
                StreamFailureHandlerFactory = partition => Task.FromResult <IStreamFailureHandler>(new NoOpStreamDeliveryFailureHandler());
            }

            if (QueueMapperFactory == null)
            {
                QueueMapperFactory = partitions => new EventHubQueueMapper(partitionIds, adapterSettings.StreamProviderName);
            }

            if (ReceiverMonitorFactory == null)
            {
                ReceiverMonitorFactory = (hubPath, partition, receiverLogger) => new DefaultEventHubReceiverMonitor(hubPath, partition, receiverLogger.GetSubLogger("monitor", "-"));
            }

            logger = log.GetLogger($"EventHub.{hubSettings.Path}");
        }
 /// <summary>
 /// Constructor for EventHubQueueCacheFactory
 /// </summary>
 /// <param name="providerSettings"></param>
 /// <param name="serializationManager"></param>
 /// <param name="sharedDimensions">shared dimensions between cache monitor and block pool monitor</param>
 /// <param name="cacheMonitorFactory"></param>
 /// <param name="blockPoolMonitorFactory"></param>
 public EventHubQueueCacheFactory(EventHubStreamProviderSettings providerSettings,
                                  SerializationManager serializationManager, EventHubMonitorAggregationDimensions sharedDimensions,
                                  Func <EventHubCacheMonitorDimensions, Logger, ITelemetryProducer, ICacheMonitor> cacheMonitorFactory             = null,
                                  Func <EventHubBlockPoolMonitorDimensions, Logger, ITelemetryProducer, IBlockPoolMonitor> blockPoolMonitorFactory = null)
 {
     this.providerSettings        = providerSettings;
     this.serializationManager    = serializationManager;
     this.timePurge               = new TimePurgePredicate(this.providerSettings.DataMinTimeInCache, this.providerSettings.DataMaxAgeInCache);
     this.sharedDimensions        = sharedDimensions;
     this.CacheMonitorFactory     = cacheMonitorFactory == null?(dimensions, logger, telemetryProducer) => new DefaultEventHubCacheMonitor(dimensions, telemetryProducer) : cacheMonitorFactory;
     this.BlockPoolMonitorFactory = blockPoolMonitorFactory == null? (dimensions, logger, telemetryProducer) => new DefaultEventHubBlockPoolMonitor(dimensions, telemetryProducer) : blockPoolMonitorFactory;
 }
 /// <summary>
 /// Constructor
 /// </summary>
 /// <param name="logger"></param>
 /// <param name="cacheMonitor"></param>
 /// <param name="monitorWriteInterval"></param>
 /// <param name="timePurage"></param>
 public EventHubCacheEvictionStrategy(Logger logger, ICacheMonitor cacheMonitor = null, TimeSpan?monitorWriteInterval = null, TimePurgePredicate timePurage = null)
 {
     this.inUseBuffers  = new Queue <FixedSizeBuffer>();
     this.logger        = logger.GetSubLogger(this.GetType().Name);
     this.purgedBuffers = new Queue <FixedSizeBuffer>();
     this.timePurge     = timePurage ?? TimePurgePredicate.Default;
     this.cacheMonitor  = cacheMonitor;
     if (cacheMonitor != null && monitorWriteInterval.HasValue)
     {
         this.timer = new Timer(this.ReportCacheSize, null, monitorWriteInterval.Value, monitorWriteInterval.Value);
     }
     this.cacheSizeInByte = 0;
 }
 /// <summary>
 /// Constructor for EventHubQueueCacheFactory
 /// </summary>
 /// <param name="cacheOptions"></param>
 /// <param name="evictionOptions"></param>
 /// <param name="statisticOptions"></param>
 /// <param name="serializationManager"></param>
 /// <param name="sharedDimensions">shared dimensions between cache monitor and block pool monitor</param>
 /// <param name="cacheMonitorFactory"></param>
 /// <param name="blockPoolMonitorFactory"></param>
 public EventHubQueueCacheFactory(
     EventHubStreamCachePressureOptions cacheOptions,
     StreamCacheEvictionOptions evictionOptions,
     StreamStatisticOptions statisticOptions,
     SerializationManager serializationManager, EventHubMonitorAggregationDimensions sharedDimensions,
     Func <EventHubCacheMonitorDimensions, ILoggerFactory, ITelemetryProducer, ICacheMonitor> cacheMonitorFactory             = null,
     Func <EventHubBlockPoolMonitorDimensions, ILoggerFactory, ITelemetryProducer, IBlockPoolMonitor> blockPoolMonitorFactory = null)
 {
     this.cacheOptions            = cacheOptions;
     this.statisticOptions        = statisticOptions;
     this.serializationManager    = serializationManager;
     this.timePurge               = new TimePurgePredicate(evictionOptions.DataMinTimeInCache, evictionOptions.DataMaxAgeInCache);
     this.sharedDimensions        = sharedDimensions;
     this.CacheMonitorFactory     = cacheMonitorFactory ?? ((dimensions, logger, telemetryProducer) => new DefaultEventHubCacheMonitor(dimensions, telemetryProducer));
     this.BlockPoolMonitorFactory = blockPoolMonitorFactory ?? ((dimensions, logger, telemetryProducer) => new DefaultEventHubBlockPoolMonitor(dimensions, telemetryProducer));
 }
        /// <summary>
        /// Default function to be called to create an EventhubQueueCache in IEventHubQueueCacheFactory.CreateCache method. User can
        /// override this method to add more customization.
        /// </summary>
        protected virtual IEventHubQueueCache CreateCache(
            string partition,
            IEventHubDataAdapter dataAdatper,
            StreamStatisticOptions statisticOptions,
            StreamCacheEvictionOptions streamCacheEvictionOptions,
            IStreamQueueCheckpointer <string> checkpointer,
            ILoggerFactory loggerFactory,
            IObjectPool <FixedSizeBuffer> bufferPool,
            string blockPoolId,
            TimePurgePredicate timePurge,
            EventHubMonitorAggregationDimensions sharedDimensions,
            ITelemetryProducer telemetryProducer)
        {
            var cacheMonitorDimensions = new EventHubCacheMonitorDimensions(sharedDimensions, partition, blockPoolId);
            var cacheMonitor           = this.CacheMonitorFactory(cacheMonitorDimensions, loggerFactory, telemetryProducer);
            var logger           = loggerFactory.CreateLogger($"{typeof(EventHubQueueCache).FullName}.{sharedDimensions.EventHubPath}.{partition}");
            var evictionStrategy = new ChronologicalEvictionStrategy(logger, timePurge, cacheMonitor, statisticOptions.StatisticMonitorWriteInterval);

            return(new EventHubQueueCache(partition, EventHubAdapterReceiver.MaxMessagesPerRead, bufferPool, dataAdatper, evictionStrategy, checkpointer, logger,
                                          cacheMonitor, statisticOptions.StatisticMonitorWriteInterval, streamCacheEvictionOptions.MetadataMinTimeInCache));
        }
        /// <summary>
        /// Default function to be called to create an EventhubQueueCache in IEventHubQueueCacheFactory.CreateCache method. User can
        /// override this method to add more customization.
        /// </summary>
        protected virtual IEventHubQueueCache CreateCache(string partition, EventHubStreamProviderSettings providerSettings, IStreamQueueCheckpointer <string> checkpointer,
                                                          Logger cacheLogger, IObjectPool <FixedSizeBuffer> bufferPool, string blockPoolId, TimePurgePredicate timePurge,
                                                          SerializationManager serializationManager, EventHubMonitorAggregationDimensions sharedDimensions, ITelemetryProducer telemetryProducer)
        {
            var cacheMonitorDimensions = new EventHubCacheMonitorDimensions(sharedDimensions, partition, blockPoolId);
            var cacheMonitor           = this.CacheMonitorFactory(cacheMonitorDimensions, cacheLogger, telemetryProducer);

            return(new EventHubQueueCache(checkpointer, bufferPool, timePurge, cacheLogger, serializationManager, cacheMonitor, providerSettings.StatisticMonitorWriteInterval));
        }
 /// <summary>
 /// Constructor
 /// </summary>
 /// <param name="logger"></param>
 /// <param name="cacheMonitor"></param>
 /// <param name="monitorWriteInterval"></param>
 /// <param name="timePurage"></param>
 public EventHubCacheEvictionStrategy(Logger logger, TimePurgePredicate timePurage, ICacheMonitor cacheMonitor, TimeSpan?monitorWriteInterval)
     : base(logger.GetLogger(LogName), timePurage, cacheMonitor, monitorWriteInterval)
 {
 }
 /// <summary>
 /// Default function to be called to create an EventhubQueueCache in IEventHubQueueCacheFactory.CreateCache method. User can
 /// override this method to add more customization.
 /// </summary>
 /// <param name="checkpointer"></param>
 /// <param name="cacheLogger"></param>
 /// <param name="bufferPool"></param>
 /// <param name="timePurge"></param>
 /// <param name="serializationManager"></param>
 /// <returns></returns>
 protected virtual IEventHubQueueCache CreateCache(IStreamQueueCheckpointer <string> checkpointer,
                                                   Logger cacheLogger, IObjectPool <FixedSizeBuffer> bufferPool, TimePurgePredicate timePurge,
                                                   SerializationManager serializationManager)
 {
     return(new EventHubQueueCache(checkpointer, bufferPool, timePurge, cacheLogger, serializationManager));
 }
        private IEventHubQueueCache CreateCacheFactory(string partition, IStreamQueueCheckpointer <string> checkpointer, Logger cacheLogger,
                                                       FixedSizeObjectPool <FixedSizeBuffer> bufferPool, TimePurgePredicate timePurge)
        {
            var cache = new EventHubQueueCache(checkpointer, bufferPool, timePurge, cacheLogger, this.SerializationManager);

            if (adapterSettings.AveragingCachePressureMonitorFlowControlThreshold.HasValue)
            {
                var avgMonitor = new AveragingCachePressureMonitor(adapterSettings.AveragingCachePressureMonitorFlowControlThreshold.Value, cacheLogger);
                cache.AddCachePressureMonitor(avgMonitor);
            }
            if (adapterSettings.SlowConsumingMonitorPressureWindowSize.HasValue ||
                adapterSettings.SlowConsumingMonitorFlowControlThreshold.HasValue)
            {
                var slowConsumeMonitor = new SlowConsumingPressureMonitor(cacheLogger);
                if (adapterSettings.SlowConsumingMonitorFlowControlThreshold.HasValue)
                {
                    slowConsumeMonitor.FlowControlThreshold = adapterSettings.SlowConsumingMonitorFlowControlThreshold.Value;
                }
                if (adapterSettings.SlowConsumingMonitorPressureWindowSize.HasValue)
                {
                    slowConsumeMonitor.PressureWindowSize = adapterSettings.SlowConsumingMonitorPressureWindowSize.Value;
                }
                cache.AddCachePressureMonitor(slowConsumeMonitor);
            }
            return(cache);
        }
        /// <summary>
        /// Default function to be called to create an EventhubQueueCache in IEventHubQueueCacheFactory.CreateCache method. User can
        /// override this method to add more customization.
        /// </summary>
        protected virtual IEventHubQueueCache CreateCache(string partition, StreamStatisticOptions statisticOptions, IStreamQueueCheckpointer <string> checkpointer,
                                                          ILoggerFactory loggerFactory, IObjectPool <FixedSizeBuffer> bufferPool, string blockPoolId, TimePurgePredicate timePurge,
                                                          SerializationManager serializationManager, EventHubMonitorAggregationDimensions sharedDimensions, ITelemetryProducer telemetryProducer)
        {
            var cacheMonitorDimensions = new EventHubCacheMonitorDimensions(sharedDimensions, partition, blockPoolId);
            var cacheMonitor           = this.CacheMonitorFactory(cacheMonitorDimensions, loggerFactory, telemetryProducer);

            return(new EventHubQueueCache(checkpointer, bufferPool, timePurge, loggerFactory.CreateLogger($"{typeof(EventHubQueueCache).FullName}.{sharedDimensions.EventHubPath}.{partition}"), serializationManager,
                                          cacheMonitor, statisticOptions.StatisticMonitorWriteInterval));
        }