protected override IEventHubQueueCacheFactory CreateCacheFactory(EventHubStreamOptions options)
{
var eventHubPath = options.Path;
var sharedDimensions = new EventHubMonitorAggregationDimensions(eventHubPath);
return(new CacheFactoryForTesting(this.Name, options, this.SerializationManager, this.createdCaches, sharedDimensions, this.serviceProvider.GetRequiredService <ILoggerFactory>()));
}
/// <summary>
/// Function used to configure cache pressure monitors for EventHubQueueCache.
/// User can override this function to provide more customization on cache pressure monitors
/// </summary>
/// <param name="cache"></param>
/// <param name="providerSettings"></param>
/// <param name="cacheLogger"></param>
protected virtual void AddCachePressureMonitors(IEventHubQueueCache cache, EventHubStreamOptions providerOptions,
ILogger cacheLogger)
{
if (providerOptions.AveragingCachePressureMonitorFlowControlThreshold.HasValue)
{
var avgMonitor = new AveragingCachePressureMonitor(
providerOptions.AveragingCachePressureMonitorFlowControlThreshold.Value, cacheLogger);
cache.AddCachePressureMonitor(avgMonitor);
}
if (providerOptions.SlowConsumingMonitorPressureWindowSize.HasValue ||
providerOptions.SlowConsumingMonitorFlowControlThreshold.HasValue)
{
var slowConsumeMonitor = new SlowConsumingPressureMonitor(cacheLogger);
if (providerOptions.SlowConsumingMonitorFlowControlThreshold.HasValue)
{
slowConsumeMonitor.FlowControlThreshold = providerOptions.SlowConsumingMonitorFlowControlThreshold.Value;
}
if (providerOptions.SlowConsumingMonitorPressureWindowSize.HasValue)
{
slowConsumeMonitor.PressureWindowSize = providerOptions.SlowConsumingMonitorPressureWindowSize.Value;
}
cache.AddCachePressureMonitor(slowConsumeMonitor);
}
}
/// <summary>
/// Create a IEventHubQueueCacheFactory. It will create a EventHubQueueCacheFactory by default.
/// User can override this function to return their own implementation of IEventHubQueueCacheFactory,
/// and other customization of IEventHubQueueCacheFactory if they may.
/// </summary>
/// <param name="providerSettings"></param>
/// <returns></returns>
protected virtual IEventHubQueueCacheFactory CreateCacheFactory(EventHubStreamOptions eventHubStreamOptions)
{
var eventHubPath = this.options.Path;
var sharedDimensions = new EventHubMonitorAggregationDimensions(eventHubPath);
return(new EventHubQueueCacheFactory(eventHubStreamOptions, this.SerializationManager, sharedDimensions, this.loggerFactory));
}
public CustomCacheFactory(string name, EventHubStreamOptions options, SerializationManager serializationManager)
{
this.name = name;
this.options = options;
this.serializationManager = serializationManager;
timePurgePredicate = new TimePurgePredicate(options.DataMinTimeInCache, options.DataMaxAgeInCache);
}
/// <summary>
/// Factory function that creates and initializes the checkpointer
/// </summary>
/// <param name="settings"></param>
/// <param name="streamProviderName"></param>
/// <param name="partition"></param>
/// <param name="loggerFactory"></param>
/// <returns></returns>
public static async Task <IStreamQueueCheckpointer <string> > Create(EventHubStreamOptions options, string streamProviderName, string partition, ILoggerFactory loggerFactory)
{
var checkpointer = new EventHubCheckpointer(options, streamProviderName, partition, loggerFactory);
await checkpointer.Initialize();
return(checkpointer);
}
public EventHubAdapterFactory(string name, EventHubStreamOptions options, IServiceProvider serviceProvider, SerializationManager serializationManager, ITelemetryProducer telemetryProducer, ILoggerFactory loggerFactory)
{
this.Name = name;
this.options = options ?? throw new ArgumentNullException(nameof(options));
this.serviceProvider = serviceProvider ?? throw new ArgumentNullException(nameof(serviceProvider));
this.SerializationManager = serializationManager ?? throw new ArgumentNullException(nameof(serializationManager));
this.telemetryProducer = telemetryProducer ?? throw new ArgumentNullException(nameof(telemetryProducer));
this.loggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory));
}
/// <summary>
/// Default function to be called to create an EventhubQueueCache in IEventHubQueueCacheFactory.CreateCache method. User can
/// override this method to add more customization.
/// </summary>
protected virtual IEventHubQueueCache CreateCache(string partition, EventHubStreamOptions providerOptions, IStreamQueueCheckpointer <string> checkpointer,
ILoggerFactory loggerFactory, IObjectPool <FixedSizeBuffer> bufferPool, string blockPoolId, TimePurgePredicate timePurge,
SerializationManager serializationManager, EventHubMonitorAggregationDimensions sharedDimensions, ITelemetryProducer telemetryProducer)
{
var cacheMonitorDimensions = new EventHubCacheMonitorDimensions(sharedDimensions, partition, blockPoolId);
var cacheMonitor = this.CacheMonitorFactory(cacheMonitorDimensions, loggerFactory, telemetryProducer);
return(new EventHubQueueCache(checkpointer, bufferPool, timePurge, loggerFactory.CreateLogger($"{typeof(EventHubQueueCache).FullName}.{sharedDimensions.EventHubPath}.{partition}"), serializationManager, cacheMonitor, providerOptions.StatisticMonitorWriteInterval));
}
public CacheFactoryForTesting(string name, EventHubStreamOptions options,
SerializationManager serializationManager, ConcurrentBag <QueueCacheForTesting> caches, EventHubMonitorAggregationDimensions sharedDimensions,
ILoggerFactory loggerFactory,
Func <EventHubCacheMonitorDimensions, ILoggerFactory, ITelemetryProducer, ICacheMonitor> cacheMonitorFactory = null,
Func <EventHubBlockPoolMonitorDimensions, ILoggerFactory, ITelemetryProducer, IBlockPoolMonitor> blockPoolMonitorFactory = null)
: base(options, serializationManager, sharedDimensions, loggerFactory, cacheMonitorFactory, blockPoolMonitorFactory)
{
this.name = name;
this.caches = caches;
}
protected override IEventHubQueueCacheFactory CreateCacheFactory(EventHubStreamOptions options)
{
var loggerFactory = this.serviceProvider.GetRequiredService <ILoggerFactory>();
var eventHubPath = options.Path;
var sharedDimensions = new EventHubMonitorAggregationDimensions(eventHubPath);
Func <EventHubCacheMonitorDimensions, ILoggerFactory, ITelemetryProducer, ICacheMonitor> cacheMonitorFactory = (dimensions, logger, telemetryProducer) => CacheMonitorForTesting.Instance;
Func <EventHubBlockPoolMonitorDimensions, ILoggerFactory, ITelemetryProducer, IBlockPoolMonitor> blockPoolMonitorFactory = (dimensions, logger, telemetryProducer) => BlockPoolMonitorForTesting.Instance;
return(new CacheFactoryForMonitorTesting(this.cachePressureInjectionMonitor, options, this.SerializationManager,
sharedDimensions, loggerFactory, cacheMonitorFactory, blockPoolMonitorFactory));
}
/// <summary>
/// Constructor for EventHubQueueCacheFactory
/// </summary>
/// <param name="options"></param>
/// <param name="serializationManager"></param>
/// <param name="sharedDimensions">shared dimensions between cache monitor and block pool monitor</param>
/// <param name="cacheMonitorFactory"></param>
/// <param name="blockPoolMonitorFactory"></param>
public EventHubQueueCacheFactory(EventHubStreamOptions options,
SerializationManager serializationManager, EventHubMonitorAggregationDimensions sharedDimensions,
ILoggerFactory loggerFactory,
Func <EventHubCacheMonitorDimensions, ILoggerFactory, ITelemetryProducer, ICacheMonitor> cacheMonitorFactory = null,
Func <EventHubBlockPoolMonitorDimensions, ILoggerFactory, ITelemetryProducer, IBlockPoolMonitor> blockPoolMonitorFactory = null)
{
this.options = options;
this.serializationManager = serializationManager;
this.timePurge = new TimePurgePredicate(this.options.DataMinTimeInCache, this.options.DataMaxAgeInCache);
this.sharedDimensions = sharedDimensions;
this.CacheMonitorFactory = cacheMonitorFactory == null?(dimensions, logger, telemetryProducer) => new DefaultEventHubCacheMonitor(dimensions, telemetryProducer) : cacheMonitorFactory;
this.BlockPoolMonitorFactory = blockPoolMonitorFactory == null? (dimensions, logger, telemetryProducer) => new DefaultEventHubBlockPoolMonitor(dimensions, telemetryProducer) : blockPoolMonitorFactory;
}
/// <summary>
/// Function used to configure BufferPool for EventHubQueueCache. User can override this function to provide more customization on BufferPool creation
/// </summary>
protected virtual IObjectPool <FixedSizeBuffer> CreateBufferPool(EventHubStreamOptions providerOptions, ILoggerFactory loggerFactory, EventHubMonitorAggregationDimensions sharedDimensions, ITelemetryProducer telemetryProducer, out string blockPoolId)
{
if (this.bufferPool == null)
{
var bufferSize = 1 << 20;
this.bufferPoolId = $"BlockPool-{new Guid().ToString()}-BlockSize-{bufferSize}";
var monitorDimensions = new EventHubBlockPoolMonitorDimensions(sharedDimensions, this.bufferPoolId);
var objectPoolMonitor = new ObjectPoolMonitorBridge(this.BlockPoolMonitorFactory(monitorDimensions, loggerFactory, telemetryProducer), bufferSize);
this.bufferPool = new ObjectPool <FixedSizeBuffer>(() => new FixedSizeBuffer(bufferSize),
objectPoolMonitor, providerOptions.StatisticMonitorWriteInterval);
}
blockPoolId = this.bufferPoolId;
return(this.bufferPool);
}
protected override IEventHubQueueCache CreateCache(string partition, EventHubStreamOptions options, IStreamQueueCheckpointer <string> checkpointer,
ILoggerFactory loggerFactory, IObjectPool <FixedSizeBuffer> bufferPool, string blockPoolId, TimePurgePredicate timePurge,
SerializationManager serializationManager, EventHubMonitorAggregationDimensions sharedDimensions, ITelemetryProducer telemetryProducer)
{
var cacheMonitorDimensions = new EventHubCacheMonitorDimensions(sharedDimensions, partition, blockPoolId);
var cacheMonitor = this.CacheMonitorFactory(cacheMonitorDimensions, loggerFactory, telemetryProducer);
var cacheLogger = loggerFactory.CreateLogger($"{typeof(EventHubQueueCache).FullName}.{this.name}.{partition}");
//set defaultMaxAddCount to 10 so TryCalculateCachePressureContribution will start to calculate real contribution shortly
var cache = new QueueCacheForTesting(DefaultMaxAddCount, checkpointer, new EventHubDataAdapter(serializationManager, bufferPool),
EventHubDataComparer.Instance, cacheLogger, new EventHubCacheEvictionStrategy(cacheLogger, timePurge, cacheMonitor, options.StatisticMonitorWriteInterval),
cacheMonitor, options.StatisticMonitorWriteInterval);
this.caches.Add(cache);
return(cache);
}
private EventHubCheckpointer(EventHubStreamOptions options, string streamProviderName, string partition, ILoggerFactory loggerFactory)
{
if (options == null)
{
throw new ArgumentNullException(nameof(options));
}
if (string.IsNullOrWhiteSpace(streamProviderName))
{
throw new ArgumentNullException(nameof(streamProviderName));
}
if (string.IsNullOrWhiteSpace(partition))
{
throw new ArgumentNullException(nameof(partition));
}
persistInterval = options.CheckpointPersistInterval;
dataManager = new AzureTableDataManager <EventHubPartitionCheckpointEntity>(options.CheckpointTableName, options.CheckpointConnectionString, loggerFactory);
entity = EventHubPartitionCheckpointEntity.Create(streamProviderName, options.CheckpointNamespace, partition);
}
protected override void AddCachePressureMonitors(IEventHubQueueCache cache, EventHubStreamOptions options,
ILogger cacheLogger)
{
cache.AddCachePressureMonitor(this.cachePressureInjectionMonitor);
}
public CacheFactoryForMonitorTesting(CachePressureInjectionMonitor cachePressureInjectionMonitor, EventHubStreamOptions options,
SerializationManager serializationManager, EventHubMonitorAggregationDimensions sharedDimensions,
ILoggerFactory loggerFactory,
Func <EventHubCacheMonitorDimensions, ILoggerFactory, ITelemetryProducer, ICacheMonitor> cacheMonitorFactory = null,
Func <EventHubBlockPoolMonitorDimensions, ILoggerFactory, ITelemetryProducer, IBlockPoolMonitor> blockPoolMonitorFactory = null)
: base(options, serializationManager, sharedDimensions, loggerFactory, cacheMonitorFactory, blockPoolMonitorFactory)
{
this.cachePressureInjectionMonitor = cachePressureInjectionMonitor;
}
protected override IEventHubQueueCacheFactory CreateCacheFactory(EventHubStreamOptions options)
{
return(new CustomCacheFactory(this.Name, options, SerializationManager));
}
public TestEventHubStreamAdapterFactory(string name, EventHubStreamOptions options, IServiceProvider serviceProvider, SerializationManager serializationManager, ITelemetryProducer telemetryProducer, ILoggerFactory loggerFactory)
: base(name, options, serviceProvider, serializationManager, telemetryProducer, loggerFactory)
{
StreamFailureHandlerFactory = qid => TestAzureTableStorageStreamFailureHandler.Create(this.serviceProvider.GetRequiredService <SerializationManager>());
}
public StreamPerPartitionEventHubStreamAdapterFactory(string name, EventHubStreamOptions options, IServiceProvider serviceProvider, SerializationManager serializationManager, ITelemetryProducer telemetryProducer, ILoggerFactory loggerFactory)
: base(name, options, serviceProvider, serializationManager, telemetryProducer, loggerFactory)
{
}
