public CustomCacheFactory(string name, StreamCacheEvictionOptions options, SerializationManager serializationManager) { this.name = name; this.options = options; this.serializationManager = serializationManager; timePurgePredicate = new TimePurgePredicate(options.DataMinTimeInCache, options.DataMaxAgeInCache); }
/// <summary> /// Factory initialization. /// </summary> /// <param name="providerConfig"></param> /// <param name="name"></param> /// <param name="svcProvider"></param> public void Init(IProviderConfiguration providerConfig, string name, IServiceProvider svcProvider) { logger = svcProvider.GetService <ILogger <MemoryAdapterFactory <TSerializer> > >(); this.loggerFactory = svcProvider.GetRequiredService <ILoggerFactory>(); serviceProvider = svcProvider; providerName = name; queueGrains = new ConcurrentDictionary <QueueId, IMemoryStreamQueueGrain>(); adapterConfig = new MemoryAdapterConfig(providerName); this.telemetryProducer = svcProvider.GetService <ITelemetryProducer>(); if (CacheMonitorFactory == null) { this.CacheMonitorFactory = (dimensions, telemetryProducer) => new DefaultCacheMonitor(dimensions, telemetryProducer); } if (this.BlockPoolMonitorFactory == null) { this.BlockPoolMonitorFactory = (dimensions, telemetryProducer) => new DefaultBlockPoolMonitor(dimensions, telemetryProducer); } if (this.ReceiverMonitorFactory == null) { this.ReceiverMonitorFactory = (dimensions, telemetryProducer) => new DefaultQueueAdapterReceiverMonitor(dimensions, telemetryProducer); } purgePredicate = new TimePurgePredicate(adapterConfig.DataMinTimeInCache, adapterConfig.DataMaxAgeInCache); grainFactory = (IGrainFactory)serviceProvider.GetService(typeof(IGrainFactory)); adapterConfig.PopulateFromProviderConfig(providerConfig); streamQueueMapper = new HashRingBasedStreamQueueMapper(adapterConfig.TotalQueueCount, adapterConfig.StreamProviderName); this.sharedDimensions = new MonitorAggregationDimensions(serviceProvider.GetService <GlobalConfiguration>(), serviceProvider.GetService <NodeConfiguration>()); this.serializer = MemoryMessageBodySerializerFactory <TSerializer> .GetOrCreateSerializer(svcProvider); }
/// <summary> /// Pooled cache for memory stream provider /// </summary> /// <param name="bufferPool"></param> /// <param name="purgePredicate"></param> /// <param name="logger"></param> /// <param name="serializer"></param> /// <param name="cacheMonitor"></param> /// <param name="monitorWriteInterval">monitor write interval. Only triggered for active caches.</param> public MemoryPooledCache(IObjectPool <FixedSizeBuffer> bufferPool, TimePurgePredicate purgePredicate, ILogger logger, TSerializer serializer, ICacheMonitor cacheMonitor, TimeSpan?monitorWriteInterval) { this.bufferPool = bufferPool; this.serializer = serializer; this.cache = new PooledQueueCache(this, logger, cacheMonitor, monitorWriteInterval); this.evictionStrategy = new ChronologicalEvictionStrategy(logger, purgePredicate, cacheMonitor, monitorWriteInterval) { PurgeObservable = cache }; }
private IEventHubQueueCache CreateQueueCache(string partition, IStreamQueueCheckpointer <string> checkpointer, Logger log) { if (timePurgePredicate != null) { timePurgePredicate = new TimePurgePredicate(adapterConfig.DataMinTimeInCache, adapterConfig.DataMaxAgeInCache); } var bufferPool = new FixedSizeObjectPool <FixedSizeBuffer>(adapterConfig.CacheSizeMb, () => new FixedSizeBuffer(1 << 20)); var dataAdapter = new CachedDataAdapter(partition, bufferPool, timePurgePredicate); return(new EventHubQueueCache(checkpointer, dataAdapter, log)); }
/// <summary> /// Pooled cache for memory stream provider /// </summary> /// <param name="bufferPool"></param> /// <param name="purgePredicate"></param> /// <param name="logger"></param> /// <param name="serializer"></param> /// <param name="cacheMonitor"></param> /// <param name="monitorWriteInterval">monitor write interval. Only triggered for active caches.</param> public MemoryPooledCache(IObjectPool <FixedSizeBuffer> bufferPool, TimePurgePredicate purgePredicate, Logger logger, TSerializer serializer, ICacheMonitor cacheMonitor, TimeSpan?monitorWriteInterval) { var dataAdapter = new CacheDataAdapter(bufferPool, serializer); cache = new PooledQueueCache <MemoryMessageData, MemoryMessageData>(dataAdapter, CacheDataComparer.Instance, logger, cacheMonitor, monitorWriteInterval); this.evictionStrategy = new MemoryPooledCacheEvictionStrategy(logger, purgePredicate, cacheMonitor, monitorWriteInterval) { PurgeObservable = cache }; EvictionStrategyCommonUtils.WireUpEvictionStrategy <MemoryMessageData, MemoryMessageData>(cache, dataAdapter, evictionStrategy); }
private IEventHubQueueCache CreateQueueCache(string partition, IStreamQueueCheckpointer <string> checkpointer, Logger log) { if (timePurgePredicate != null) { timePurgePredicate = new TimePurgePredicate(adapterSettings.DataMinTimeInCache, adapterSettings.DataMaxAgeInCache); } var bufferPool = new FixedSizeObjectPool <FixedSizeBuffer>(adapterSettings.CacheSizeMb, () => new FixedSizeBuffer(1 << 20)); var dataAdapter = new CachedDataAdapter(partition, bufferPool, timePurgePredicate, this.SerializationManager); return(new EventHubQueueCache(checkpointer, dataAdapter, EventHubDataComparer.Instance, log)); }
/// <summary> /// Pooled cache for generator stream provider /// </summary> /// <param name="bufferPool"></param> /// <param name="logger"></param> /// <param name="serializationManager"></param> /// <param name="cacheMonitor"></param> /// <param name="monitorWriteInterval">monitor write interval. Only triggered for active caches.</param> public GeneratorPooledCache(IObjectPool <FixedSizeBuffer> bufferPool, ILogger logger, SerializationManager serializationManager, ICacheMonitor cacheMonitor, TimeSpan?monitorWriteInterval) { this.bufferPool = bufferPool; this.serializationManager = serializationManager; cache = new PooledQueueCache(this, logger, cacheMonitor, monitorWriteInterval); TimePurgePredicate purgePredicate = new TimePurgePredicate(TimeSpan.FromMinutes(5), TimeSpan.FromMinutes(10)); this.evictionStrategy = new ChronologicalEvictionStrategy(logger, purgePredicate, cacheMonitor, monitorWriteInterval) { PurgeObservable = cache }; }
/// <summary> /// Pooled cache for generator stream provider /// </summary> /// <param name="bufferPool"></param> /// <param name="logger"></param> /// <param name="serializationManager"></param> /// <param name="cacheMonitor"></param> /// <param name="monitorWriteInterval"></param> public GeneratorPooledCache(IObjectPool <FixedSizeBuffer> bufferPool, Logger logger, SerializationManager serializationManager, ICacheMonitor cacheMonitor, TimeSpan?monitorWriteInterval) { var dataAdapter = new CacheDataAdapter(bufferPool, serializationManager); cache = new PooledQueueCache <GeneratedBatchContainer, CachedMessage>(dataAdapter, CacheDataComparer.Instance, logger, cacheMonitor, monitorWriteInterval); TimePurgePredicate purgePredicate = new TimePurgePredicate(TimeSpan.FromMinutes(5), TimeSpan.FromMinutes(10)); this.evictionStrategy = new GeneratorPooledCacheEvictionStrategy(logger, purgePredicate, cacheMonitor, monitorWriteInterval) { PurgeObservable = cache }; EvictionStrategyCommonUtils.WireUpEvictionStrategy(cache, dataAdapter, this.evictionStrategy); }
/// <summary> /// Factory initialization. /// </summary> public void Init() { this.queueGrains = new ConcurrentDictionary <QueueId, IMemoryStreamQueueGrain>(); if (CacheMonitorFactory == null) { this.CacheMonitorFactory = (dimensions, telemetryProducer) => new DefaultCacheMonitor(dimensions, telemetryProducer); } if (this.BlockPoolMonitorFactory == null) { this.BlockPoolMonitorFactory = (dimensions, telemetryProducer) => new DefaultBlockPoolMonitor(dimensions, telemetryProducer); } if (this.ReceiverMonitorFactory == null) { this.ReceiverMonitorFactory = (dimensions, telemetryProducer) => new DefaultQueueAdapterReceiverMonitor(dimensions, telemetryProducer); } this.purgePredicate = new TimePurgePredicate(this.cacheOptions.DataMinTimeInCache, this.cacheOptions.DataMaxAgeInCache); this.streamQueueMapper = new HashRingBasedStreamQueueMapper(this.queueMapperOptions, this.Name); }
private IEventHubQueueCache CreateQueueCache(string partition, IStreamQueueCheckpointer <string> checkpointer, Logger cacheLogger) { // the same code block as with EventHubAdapterFactory to define default CacheFactory // except for at the end we put the created cache in CreatedCaches list if (this.bufferPool == null) { this.bufferPool = new FixedSizeObjectPool <FixedSizeBuffer>(adapterSettings.CacheSizeMb, () => new FixedSizeBuffer(1 << 20)); } if (this.timePurge == null) { this.timePurge = new TimePurgePredicate(adapterSettings.DataMinTimeInCache, adapterSettings.DataMaxAgeInCache); } //ser defaultMaxAddCount to 10 so TryCalculateCachePressureContribution will start to calculate real contribution shortly. var cache = new EventHubQueueCache(defaultMaxAddCount, checkpointer, new EventHubDataAdapter(this.SerializationManager, bufferPool, timePurge), EventHubDataComparer.Instance, cacheLogger); if (adapterSettings.AveragingCachePressureMonitorFlowControlThreshold.HasValue) { var avgMonitor = new AveragingCachePressureMonitor(adapterSettings.AveragingCachePressureMonitorFlowControlThreshold.Value, cacheLogger); cache.AddCachePressureMonitor(avgMonitor); } if (adapterSettings.SlowConsumingMonitorPressureWindowSize.HasValue || adapterSettings.SlowConsumingMonitorFlowControlThreshold.HasValue) { var slowConsumeMonitor = new SlowConsumingPressureMonitor(cacheLogger); if (adapterSettings.SlowConsumingMonitorFlowControlThreshold.HasValue) { slowConsumeMonitor.FlowControlThreshold = adapterSettings.SlowConsumingMonitorFlowControlThreshold.Value; } if (adapterSettings.SlowConsumingMonitorPressureWindowSize.HasValue) { slowConsumeMonitor.PressureWindowSize = adapterSettings.SlowConsumingMonitorPressureWindowSize.Value; } cache.AddCachePressureMonitor(slowConsumeMonitor); } this.createdCaches.Add(cache); return(cache); }
protected override IEventHubQueueCache CreateCache( string partition, IEventHubDataAdapter dataAdatper, StreamStatisticOptions options, StreamCacheEvictionOptions evictionOptions, IStreamQueueCheckpointer <string> checkpointer, ILoggerFactory loggerFactory, IObjectPool <FixedSizeBuffer> bufferPool, string blockPoolId, TimePurgePredicate timePurge, EventHubMonitorAggregationDimensions sharedDimensions, ITelemetryProducer telemetryProducer) { var cacheMonitorDimensions = new EventHubCacheMonitorDimensions(sharedDimensions, partition, blockPoolId); var cacheMonitor = this.CacheMonitorFactory(cacheMonitorDimensions, loggerFactory, telemetryProducer); var cacheLogger = loggerFactory.CreateLogger($"{typeof(EventHubQueueCache).FullName}.{this.name}.{partition}"); var evictionStrategy = new ChronologicalEvictionStrategy(cacheLogger, timePurge, cacheMonitor, options.StatisticMonitorWriteInterval); //set defaultMaxAddCount to 10 so TryCalculateCachePressureContribution will start to calculate real contribution shortly var cache = new QueueCacheForTesting(DefaultMaxAddCount, bufferPool, dataAdatper, evictionStrategy, checkpointer, cacheLogger, cacheMonitor, options.StatisticMonitorWriteInterval); this.caches.Add(cache); return(cache); }
protected override IEventHubQueueCache CreateCache(IStreamQueueCheckpointer <string> checkpointer, Logger cacheLogger, IObjectPool <FixedSizeBuffer> bufferPool, TimePurgePredicate timePurge, SerializationManager serializationManager) { //set defaultMaxAddCount to 10 so TryCalculateCachePressureContribution will start to calculate real contribution shortly. var cache = new EventHubQueueCache(defaultMaxAddCount, checkpointer, new EventHubDataAdapter(serializationManager, bufferPool), EventHubDataComparer.Instance, cacheLogger, new EventHubCacheEvictionStrategy(cacheLogger, timePurge)); _caches.Add(cache); return(cache); }
public CachedDataAdapter(string partitionKey, IObjectPool <FixedSizeBuffer> bufferPool, TimePurgePredicate timePurge) : base(bufferPool, timePurge) { partitionStreamGuid = GetPartitionGuid(partitionKey); }
public EHEvictionStrategyForTesting(Logger logger, TimePurgePredicate timePurage = null) : base(logger, timePurage) { }
protected override IEventHubQueueCache CreateCache(string partition, EventHubStreamProviderSettings providerSettings, IStreamQueueCheckpointer <string> checkpointer, Logger cacheLogger, IObjectPool <FixedSizeBuffer> bufferPool, string blockPoolId, TimePurgePredicate timePurge, SerializationManager serializationManager, EventHubMonitorAggregationDimensions sharedDimensions) { var cacheMonitorDimensions = new EventHubCacheMonitorDimensions(sharedDimensions, partition, blockPoolId); var cacheMonitor = this.CacheMonitorFactory(cacheMonitorDimensions, cacheLogger); //set defaultMaxAddCount to 10 so TryCalculateCachePressureContribution will start to calculate real contribution shortly var cache = new EventHubQueueCache(defaultMaxAddCount, checkpointer, new EventHubDataAdapter(serializationManager, bufferPool), EventHubDataComparer.Instance, cacheLogger, new EventHubCacheEvictionStrategy(cacheLogger, cacheMonitor, providerSettings.StatisticMonitorWriteInterval, timePurge), cacheMonitor, providerSettings.StatisticMonitorWriteInterval); this.caches.Add(cache); return(cache); }
public CacheFactory(EventHubStreamProviderSettings adapterSettings, SerializationManager serializationManager) { this.adapterSettings = adapterSettings; this.serializationManager = serializationManager; timePurgePredicate = new TimePurgePredicate(adapterSettings.DataMinTimeInCache, adapterSettings.DataMaxAgeInCache); }
public EHEvictionStrategyForTesting(ILogger logger, ICacheMonitor cacheMonitor = null, TimeSpan?monitorWriteInterval = null, TimePurgePredicate timePurage = null) : base(logger, timePurage, cacheMonitor, monitorWriteInterval) { }
public GeneratorPooledCacheEvictionStrategy(Logger logger, TimePurgePredicate purgePredicate, ICacheMonitor cacheMonitor, TimeSpan?monitorWriteInterval) : base(logger, purgePredicate, cacheMonitor, monitorWriteInterval) { }
protected override IEventHubQueueCache CreateCache(IStreamQueueCheckpointer <string> checkpointer, Logger cacheLogger, IObjectPool <FixedSizeBuffer> bufferPool, TimePurgePredicate timePurge, SerializationManager serializationManager) { var cache = base.CreateCache(checkpointer, cacheLogger, bufferPool, timePurge, serializationManager); _caches.Add(cache); return(cache); }