/// <summary>
/// Create a IEventHubQueueCacheFactory. It will create a EventHubQueueCacheFactory by default.
/// User can override this function to return their own implementation of IEventHubQueueCacheFactory,
/// and other customization of IEventHubQueueCacheFactory if they may.
/// </summary>
/// <param name="providerSettings"></param>
/// <returns></returns>
protected virtual IEventHubQueueCacheFactory CreateCacheFactory(EventHubStreamProviderSettings providerSettings)
{
var eventHubPath = this.hubSettings.Path;
var sharedDimensions = new EventHubMonitorAggregationDimensions(eventHubPath);
return(new EventHubQueueCacheFactory(providerSettings, this.SerializationManager, sharedDimensions, this.loggerFactory));
}
/// <summary>
/// Function used to configure cache pressure monitors for EventHubQueueCache.
/// User can override this function to provide more customization on cache pressure monitors
/// </summary>
/// <param name="cache"></param>
/// <param name="providerSettings"></param>
/// <param name="cacheLogger"></param>
protected virtual void AddCachePressureMonitors(IEventHubQueueCache cache, EventHubStreamProviderSettings providerSettings,
Logger cacheLogger)
{
if (providerSettings.AveragingCachePressureMonitorFlowControlThreshold.HasValue)
{
var avgMonitor = new AveragingCachePressureMonitor(
providerSettings.AveragingCachePressureMonitorFlowControlThreshold.Value, cacheLogger);
cache.AddCachePressureMonitor(avgMonitor);
}
if (providerSettings.SlowConsumingMonitorPressureWindowSize.HasValue ||
providerSettings.SlowConsumingMonitorFlowControlThreshold.HasValue)
{
var slowConsumeMonitor = new SlowConsumingPressureMonitor(cacheLogger);
if (providerSettings.SlowConsumingMonitorFlowControlThreshold.HasValue)
{
slowConsumeMonitor.FlowControlThreshold = providerSettings.SlowConsumingMonitorFlowControlThreshold.Value;
}
if (providerSettings.SlowConsumingMonitorPressureWindowSize.HasValue)
{
slowConsumeMonitor.PressureWindowSize = providerSettings.SlowConsumingMonitorPressureWindowSize.Value;
}
cache.AddCachePressureMonitor(slowConsumeMonitor);
}
}
/// <summary>
/// Constructor for EventHubQueueCacheFactory
/// </summary>
/// <param name="providerSettings"></param>
/// <param name="serializationManager"></param>
public EventHubQueueCacheFactory(EventHubStreamProviderSettings providerSettings,
SerializationManager serializationManager
)
{
_providerSettings = providerSettings;
_serializationManager = serializationManager;
_timePurge = new TimePurgePredicate(_providerSettings.DataMinTimeInCache, _providerSettings.DataMaxAgeInCache);
}
/// <summary>
/// Default function to be called to create an EventhubQueueCache in IEventHubQueueCacheFactory.CreateCache method. User can
/// override this method to add more customization.
/// </summary>
protected virtual IEventHubQueueCache CreateCache(string partition, EventHubStreamProviderSettings providerSettings, IStreamQueueCheckpointer <string> checkpointer,
Logger cacheLogger, IObjectPool <FixedSizeBuffer> bufferPool, string blockPoolId, TimePurgePredicate timePurge,
SerializationManager serializationManager, EventHubMonitorAggregationDimensions sharedDimensions, ITelemetryProducer telemetryProducer)
{
var cacheMonitorDimensions = new EventHubCacheMonitorDimensions(sharedDimensions, partition, blockPoolId);
var cacheMonitor = this.CacheMonitorFactory(cacheMonitorDimensions, cacheLogger, telemetryProducer);
return(new EventHubQueueCache(checkpointer, bufferPool, timePurge, cacheLogger, serializationManager, cacheMonitor, providerSettings.StatisticMonitorWriteInterval));
}
/// <summary>
/// Create a IEventHubQueueCacheFactory. It will create a EventHubQueueCacheFactory by default.
/// User can override this function to return their own implementation of IEventHubQueueCacheFactory,
/// and other customization of IEventHubQueueCacheFactory if they may.
/// </summary>
/// <param name="providerSettings"></param>
/// <returns></returns>
protected virtual IEventHubQueueCacheFactory CreateCacheFactory(EventHubStreamProviderSettings providerSettings)
{
var globalConfig = this.serviceProvider.GetService <GlobalConfiguration>();
var nodeConfig = this.serviceProvider.GetService <NodeConfiguration>();
var eventHubPath = hubSettings.Path;
var sharedDimensions = new EventHubMonitorAggregationDimensions(globalConfig, nodeConfig, eventHubPath);
return(new EventHubQueueCacheFactory(providerSettings, SerializationManager, sharedDimensions));
}
/// <summary>
/// Factory initialization.
/// Provider config must contain the event hub settings type or the settings themselves.
/// EventHubSettingsType is recommended for consumers that do not want to include secure information in the cluster configuration.
/// </summary>
/// <param name="providerCfg"></param>
/// <param name="providerName"></param>
/// <param name="log"></param>
/// <param name="svcProvider"></param>
public virtual void Init(IProviderConfiguration providerCfg, string providerName, Logger log, IServiceProvider svcProvider)
{
if (providerCfg == null)
{
throw new ArgumentNullException(nameof(providerCfg));
}
if (string.IsNullOrWhiteSpace(providerName))
{
throw new ArgumentNullException(nameof(providerName));
}
if (log == null)
{
throw new ArgumentNullException(nameof(log));
}
providerConfig = providerCfg;
serviceProvider = svcProvider;
receivers = new ConcurrentDictionary <QueueId, EventHubAdapterReceiver>();
adapterSettings = new EventHubStreamProviderSettings(providerName);
adapterSettings.PopulateFromProviderConfig(providerConfig);
hubSettings = adapterSettings.GetEventHubSettings(providerConfig, serviceProvider);
client = EventHubClient.CreateFromConnectionString(hubSettings.ConnectionString, hubSettings.Path);
if (CheckpointerFactory == null)
{
checkpointerSettings = adapterSettings.GetCheckpointerSettings(providerConfig, serviceProvider);
CheckpointerFactory = partition => EventHubCheckpointer.Create(checkpointerSettings, adapterSettings.StreamProviderName, partition);
}
if (CacheFactory == null)
{
var bufferPool = new FixedSizeObjectPool <FixedSizeBuffer>(adapterSettings.CacheSizeMb, () => new FixedSizeBuffer(1 << 20));
var timePurge = new TimePurgePredicate(adapterSettings.DataMinTimeInCache, adapterSettings.DataMaxAgeInCache);
CacheFactory = (partition, checkpointer, cacheLogger) => new EventHubQueueCache(checkpointer, bufferPool, timePurge, cacheLogger);
}
if (StreamFailureHandlerFactory == null)
{
//TODO: Add a queue specific default failure handler with reasonable error reporting.
StreamFailureHandlerFactory = partition => Task.FromResult <IStreamFailureHandler>(new NoOpStreamDeliveryFailureHandler());
}
if (QueueMapperFactory == null)
{
QueueMapperFactory = partitions => new EventHubQueueMapper(partitionIds, adapterSettings.StreamProviderName);
}
if (ReceiverMonitorFactory == null)
{
ReceiverMonitorFactory = (hubPath, partition, receiverLogger) => new DefaultEventHubReceiverMonitor(hubPath, partition, receiverLogger.GetSubLogger("monitor", "-"));
}
logger = log.GetLogger($"EventHub.{hubSettings.Path}");
}
/// <summary>
/// Factory initialization.
/// Provider config must contain the event hub settings type or the settings themselves.
/// EventHubSettingsType is recommended for consumers that do not want to include secure information in the cluster configuration.
/// </summary>
/// <param name="providerCfg"></param>
/// <param name="providerName"></param>
/// <param name="log"></param>
/// <param name="svcProvider"></param>
public virtual void Init(IProviderConfiguration providerCfg, string providerName, Logger log, IServiceProvider svcProvider)
{
if (providerCfg == null)
{
throw new ArgumentNullException(nameof(providerCfg));
}
if (string.IsNullOrWhiteSpace(providerName))
{
throw new ArgumentNullException(nameof(providerName));
}
if (log == null)
{
throw new ArgumentNullException(nameof(log));
}
providerConfig = providerCfg;
serviceProvider = svcProvider;
this.loggerFactory = serviceProvider.GetRequiredService <ILoggerFactory>();
receivers = new ConcurrentDictionary <QueueId, EventHubAdapterReceiver>();
this.SerializationManager = this.serviceProvider.GetRequiredService <SerializationManager>();
adapterSettings = new EventHubStreamProviderSettings(providerName);
adapterSettings.PopulateFromProviderConfig(providerConfig);
hubSettings = adapterSettings.GetEventHubSettings(providerConfig, serviceProvider);
this.telemetryProducer = serviceProvider.GetService <ITelemetryProducer>();
InitEventHubClient();
if (CheckpointerFactory == null)
{
checkpointerSettings = adapterSettings.GetCheckpointerSettings(providerConfig, serviceProvider);
CheckpointerFactory = partition => EventHubCheckpointer.Create(checkpointerSettings, adapterSettings.StreamProviderName, partition, this.loggerFactory);
}
if (CacheFactory == null)
{
CacheFactory = CreateCacheFactory(adapterSettings).CreateCache;
}
if (StreamFailureHandlerFactory == null)
{
//TODO: Add a queue specific default failure handler with reasonable error reporting.
StreamFailureHandlerFactory = partition => Task.FromResult <IStreamFailureHandler>(new NoOpStreamDeliveryFailureHandler());
}
if (QueueMapperFactory == null)
{
QueueMapperFactory = partitions => new EventHubQueueMapper(partitions, adapterSettings.StreamProviderName);
}
if (ReceiverMonitorFactory == null)
{
ReceiverMonitorFactory = (dimensions, logger, telemetryProducer) => new DefaultEventHubReceiverMonitor(dimensions, telemetryProducer);
}
logger = log.GetLogger($"EventHub.{hubSettings.Path}");
}
/// <summary>
/// Constructor for EventHubQueueCacheFactory
/// </summary>
/// <param name="providerSettings"></param>
/// <param name="serializationManager"></param>
/// <param name="sharedDimensions">shared dimensions between cache monitor and block pool monitor</param>
/// <param name="cacheMonitorFactory"></param>
/// <param name="blockPoolMonitorFactory"></param>
public EventHubQueueCacheFactory(EventHubStreamProviderSettings providerSettings,
SerializationManager serializationManager, EventHubMonitorAggregationDimensions sharedDimensions,
Func <EventHubCacheMonitorDimensions, Logger, ITelemetryProducer, ICacheMonitor> cacheMonitorFactory = null,
Func <EventHubBlockPoolMonitorDimensions, Logger, ITelemetryProducer, IBlockPoolMonitor> blockPoolMonitorFactory = null)
{
this.providerSettings = providerSettings;
this.serializationManager = serializationManager;
this.timePurge = new TimePurgePredicate(this.providerSettings.DataMinTimeInCache, this.providerSettings.DataMaxAgeInCache);
this.sharedDimensions = sharedDimensions;
this.CacheMonitorFactory = cacheMonitorFactory == null?(dimensions, logger, telemetryProducer) => new DefaultEventHubCacheMonitor(dimensions, telemetryProducer) : cacheMonitorFactory;
this.BlockPoolMonitorFactory = blockPoolMonitorFactory == null? (dimensions, logger, telemetryProducer) => new DefaultEventHubBlockPoolMonitor(dimensions, telemetryProducer) : blockPoolMonitorFactory;
}
/// <summary>
/// Function used to configure BufferPool for EventHubQueueCache. User can override this function to provide more customization on BufferPool creation
/// </summary>
protected virtual IObjectPool <FixedSizeBuffer> CreateBufferPool(EventHubStreamProviderSettings providerSettings, Logger logger, EventHubMonitorAggregationDimensions sharedDimensions, ITelemetryProducer telemetryProducer, out string blockPoolId)
{
if (this.bufferPool == null)
{
var bufferSize = 1 << 20;
this.bufferPoolId = $"BlockPool-{new Guid().ToString()}-BlockSize-{bufferSize}";
var monitorDimensions = new EventHubBlockPoolMonitorDimensions(sharedDimensions, this.bufferPoolId);
var objectPoolMonitor = new ObjectPoolMonitorBridge(this.BlockPoolMonitorFactory(monitorDimensions, logger, telemetryProducer), bufferSize);
this.bufferPool = new ObjectPool <FixedSizeBuffer>(() => new FixedSizeBuffer(bufferSize),
objectPoolMonitor, providerSettings.StatisticMonitorWriteInterval);
}
blockPoolId = this.bufferPoolId;
return(this.bufferPool);
}
/// <summary>
/// Create a IEventHubQueueCacheFactory. It will create a EventHubQueueCacheFactory by default.
/// User can override this function to return their own implementation of IEventHubQueueCacheFactory,
/// and other customization of IEventHubQueueCacheFactory if they may.
/// </summary>
/// <param name="providerSettings"></param>
/// <returns></returns>
protected virtual IEventHubQueueCacheFactory CreateCacheFactory(EventHubStreamProviderSettings providerSettings)
{
return(new EventHubQueueCacheFactory(providerSettings, SerializationManager));
}
/// <summary>
/// Factory initialization.
/// Provider config must contain the event hub settings type or the settings themselves.
/// EventHubSettingsType is recommended for consumers that do not want to include secure information in the cluster configuration.
/// </summary>
/// <param name="providerCfg"></param>
/// <param name="providerName"></param>
/// <param name="log"></param>
/// <param name="svcProvider"></param>
public virtual void Init(IProviderConfiguration providerCfg, string providerName, Logger log, IServiceProvider svcProvider)
{
if (providerCfg == null)
{
throw new ArgumentNullException(nameof(providerCfg));
}
if (string.IsNullOrWhiteSpace(providerName))
{
throw new ArgumentNullException(nameof(providerName));
}
if (log == null)
{
throw new ArgumentNullException(nameof(log));
}
providerConfig = providerCfg;
serviceProvider = svcProvider;
receivers = new ConcurrentDictionary <QueueId, EventHubAdapterReceiver>();
this.SerializationManager = this.serviceProvider.GetRequiredService <SerializationManager>();
adapterSettings = new EventHubStreamProviderSettings(providerName);
adapterSettings.PopulateFromProviderConfig(providerConfig);
hubSettings = adapterSettings.GetEventHubSettings(providerConfig, serviceProvider);
#if NETSTANDARD
var connectionStringBuilder = new EventHubsConnectionStringBuilder(hubSettings.ConnectionString)
{
EntityPath = hubSettings.Path
};
client = EventHubClient.CreateFromConnectionString(connectionStringBuilder.ToString());
#else
client = EventHubClient.CreateFromConnectionString(hubSettings.ConnectionString, hubSettings.Path);
#endif
if (CheckpointerFactory == null)
{
checkpointerSettings = adapterSettings.GetCheckpointerSettings(providerConfig, serviceProvider);
CheckpointerFactory = partition => EventHubCheckpointer.Create(checkpointerSettings, adapterSettings.StreamProviderName, partition);
}
if (CacheFactory == null)
{
CacheFactory = CreateCacheFactory(adapterSettings).CreateCache;
}
if (StreamFailureHandlerFactory == null)
{
//TODO: Add a queue specific default failure handler with reasonable error reporting.
StreamFailureHandlerFactory = partition => Task.FromResult <IStreamFailureHandler>(new NoOpStreamDeliveryFailureHandler());
}
if (QueueMapperFactory == null)
{
QueueMapperFactory = partitions => new EventHubQueueMapper(partitionIds, adapterSettings.StreamProviderName);
}
if (ReceiverMonitorFactory == null)
{
ReceiverMonitorFactory = (hubPath, partition, receiverLogger) => new DefaultEventHubReceiverMonitor(hubPath, partition, receiverLogger.GetSubLogger("monitor", "-"));
}
logger = log.GetLogger($"EventHub.{hubSettings.Path}");
}
/// <summary>
/// Function used to configure BufferPool for EventHubQueueCache. User can override this function to provide more customization on BufferPool creation
/// </summary>
/// <param name="providerSettings"></param>
/// <returns></returns>
protected virtual IObjectPool <FixedSizeBuffer> CreateBufferPool(EventHubStreamProviderSettings providerSettings)
{
return(_bufferPool ?? (_bufferPool = new FixedSizeObjectPool <FixedSizeBuffer>(providerSettings.CacheSizeMb,
() => new FixedSizeBuffer(1 << 20))));
}