public void Alloc10Free1Test() { var accumulator = new Accumulator(); IObjectPool <TestPooledResource> pool = new FixedSizeObjectPool <TestPooledResource>(10, p => new TestPooledResource(p, accumulator)); // Allocate 10 items var resources = Enumerable.Range(0, 10).Select(i => pool.Allocate()).ToList(); // free 10 resources.ForEach(r => r.Dispose()); // pool was pre-populated with 10, so max allocated should be 10 Assert.AreEqual(10, accumulator.MaxAllocated); Assert.AreEqual(0, accumulator.CurrentlyAllocated); // Allocate and free 10 items for (int i = 0; i < 10; i++) { TestPooledResource resource = pool.Allocate(); resource.Dispose(); } // only 1 at a time was ever allocated, but pool was pre-populated with 10, so max allocated should be 10 Assert.AreEqual(10, accumulator.MaxAllocated); Assert.AreEqual(0, accumulator.CurrentlyAllocated); }
public EHPurgeLogicTests() { //an mock eh settings this.ehSettings = new EventHubPartitionSettings(); ehSettings.Hub = new EventHubSettings(); ehSettings.Partition = "MockPartition"; //set up cache pressure monitor and purge predicate this.cachePressureInjectionMonitor = new CachePressureInjectionMonitor(); this.purgePredicate = new PurgeDecisionInjectionPredicate(TimeSpan.FromMinutes(5), TimeSpan.FromMinutes(30)); //set up serialization env var environment = SerializationTestEnvironment.InitializeWithDefaults(); this.serializationManager = environment.SerializationManager; //set up buffer pool, small buffer size make it easy for cache to allocate multiple buffers this.bufferPoolSizeInMB = EventHubStreamProviderSettings.DefaultCacheSizeMb; var oneKB = 1024; this.bufferPool = new FixedSizeObjectPool <FixedSizeBuffer>(this.bufferPoolSizeInMB, () => new FixedSizeBuffer(oneKB)); //set up logger this.logger = new NoOpTestLogger().GetLogger(this.GetType().Name); }
private IEventHubQueueCache CreateQueueCache(string partition, IStreamQueueCheckpointer <string> checkpointer) { var bufferPool = new FixedSizeObjectPool <FixedSizeBuffer>(adapterConfig.CacheSizeMb, pool => new FixedSizeBuffer(1 << 20, pool)); var dataAdapter = new CachedDataAdapter(partition, bufferPool); return(new EventHubQueueCache(checkpointer, dataAdapter)); }
public IEventHubQueueCache CreateCache(string partition, IStreamQueueCheckpointer <string> checkpointer, Logger cacheLogger) { var bufferPool = new FixedSizeObjectPool <FixedSizeBuffer>(adapterSettings.CacheSizeMb, () => new FixedSizeBuffer(1 << 20)); var dataAdapter = new CachedDataAdapter(partition, bufferPool, timePurgePredicate, this.serializationManager); return(new EventHubQueueCache(checkpointer, dataAdapter, EventHubDataComparer.Instance, cacheLogger)); }
private IEventHubQueueCache CreateCacheFactory(string partition, IStreamQueueCheckpointer <string> checkpointer, Logger cacheLogger, FixedSizeObjectPool <FixedSizeBuffer> bufferPool, TimePurgePredicate timePurge) { var cache = new EventHubQueueCache(checkpointer, bufferPool, timePurge, cacheLogger, this.SerializationManager); if (adapterSettings.AveragingCachePressureMonitorFlowControlThreshold.HasValue) { var avgMonitor = new AveragingCachePressureMonitor(adapterSettings.AveragingCachePressureMonitorFlowControlThreshold.Value, cacheLogger); cache.AddCachePressureMonitor(avgMonitor); } if (adapterSettings.SlowConsumingMonitorPressureWindowSize.HasValue || adapterSettings.SlowConsumingMonitorFlowControlThreshold.HasValue) { var slowConsumeMonitor = new SlowConsumingPressureMonitor(cacheLogger); if (adapterSettings.SlowConsumingMonitorFlowControlThreshold.HasValue) { slowConsumeMonitor.FlowControlThreshold = adapterSettings.SlowConsumingMonitorFlowControlThreshold.Value; } if (adapterSettings.SlowConsumingMonitorPressureWindowSize.HasValue) { slowConsumeMonitor.PressureWindowSize = adapterSettings.SlowConsumingMonitorPressureWindowSize.Value; } cache.AddCachePressureMonitor(slowConsumeMonitor); } return(cache); }
/// <summary> /// Factory initialization. /// Provider config must contain the event hub settings type or the settings themselves. /// EventHubSettingsType is recommended for consumers that do not want to include secure information in the cluster configuration. /// </summary> /// <param name="providerCfg"></param> /// <param name="providerName"></param> /// <param name="log"></param> /// <param name="svcProvider"></param> public virtual void Init(IProviderConfiguration providerCfg, string providerName, Logger log, IServiceProvider svcProvider) { if (providerCfg == null) { throw new ArgumentNullException(nameof(providerCfg)); } if (string.IsNullOrWhiteSpace(providerName)) { throw new ArgumentNullException(nameof(providerName)); } if (log == null) { throw new ArgumentNullException(nameof(log)); } providerConfig = providerCfg; serviceProvider = svcProvider; receivers = new ConcurrentDictionary <QueueId, EventHubAdapterReceiver>(); adapterSettings = new EventHubStreamProviderSettings(providerName); adapterSettings.PopulateFromProviderConfig(providerConfig); hubSettings = adapterSettings.GetEventHubSettings(providerConfig, serviceProvider); client = EventHubClient.CreateFromConnectionString(hubSettings.ConnectionString, hubSettings.Path); if (CheckpointerFactory == null) { checkpointerSettings = adapterSettings.GetCheckpointerSettings(providerConfig, serviceProvider); CheckpointerFactory = partition => EventHubCheckpointer.Create(checkpointerSettings, adapterSettings.StreamProviderName, partition); } if (CacheFactory == null) { var bufferPool = new FixedSizeObjectPool <FixedSizeBuffer>(adapterSettings.CacheSizeMb, () => new FixedSizeBuffer(1 << 20)); var timePurge = new TimePurgePredicate(adapterSettings.DataMinTimeInCache, adapterSettings.DataMaxAgeInCache); CacheFactory = (partition, checkpointer, cacheLogger) => new EventHubQueueCache(checkpointer, bufferPool, timePurge, cacheLogger); } if (StreamFailureHandlerFactory == null) { //TODO: Add a queue specific default failure handler with reasonable error reporting. StreamFailureHandlerFactory = partition => Task.FromResult <IStreamFailureHandler>(new NoOpStreamDeliveryFailureHandler()); } if (QueueMapperFactory == null) { QueueMapperFactory = partitions => new EventHubQueueMapper(partitionIds, adapterSettings.StreamProviderName); } if (ReceiverMonitorFactory == null) { ReceiverMonitorFactory = (hubPath, partition, receiverLogger) => new DefaultEventHubReceiverMonitor(hubPath, partition, receiverLogger.GetSubLogger("monitor", "-")); } logger = log.GetLogger($"EventHub.{hubSettings.Path}"); }
private IEventHubQueueCache CreateQueueCache(string partition, IStreamQueueCheckpointer <string> checkpointer, Logger log) { if (timePurgePredicate != null) { timePurgePredicate = new TimePurgePredicate(adapterConfig.DataMinTimeInCache, adapterConfig.DataMaxAgeInCache); } var bufferPool = new FixedSizeObjectPool <FixedSizeBuffer>(adapterConfig.CacheSizeMb, () => new FixedSizeBuffer(1 << 20)); var dataAdapter = new CachedDataAdapter(partition, bufferPool, timePurgePredicate); return(new EventHubQueueCache(checkpointer, dataAdapter, log)); }
private IEventHubQueueCache CreateQueueCache(string partition, IStreamQueueCheckpointer <string> checkpointer, Logger log) { if (timePurgePredicate != null) { timePurgePredicate = new TimePurgePredicate(adapterSettings.DataMinTimeInCache, adapterSettings.DataMaxAgeInCache); } var bufferPool = new FixedSizeObjectPool <FixedSizeBuffer>(adapterSettings.CacheSizeMb, () => new FixedSizeBuffer(1 << 20)); var dataAdapter = new CachedDataAdapter(partition, bufferPool, timePurgePredicate, this.SerializationManager); return(new EventHubQueueCache(checkpointer, dataAdapter, EventHubDataComparer.Instance, log)); }
public void Alloc50Max10Test() { var accumulator = new Accumulator(); IObjectPool <TestPooledResource> pool = new FixedSizeObjectPool <TestPooledResource>(10, () => new TestPooledResource(accumulator)); // Allocate 50 items var resources = Enumerable.Range(0, 50).Select(i => pool.Allocate()).ToList(); // pool was set to max of 10, so max allocated should be 10 Assert.Equal(10, accumulator.MaxAllocated); }
/// <summary> /// Factory initialization. /// Provider config must contain the event hub settings type or the settings themselves. /// EventHubSettingsType is recommended for consumers that do not want to include secure information in the cluster configuration. /// </summary> /// <param name="providerConfig"></param> /// <param name="providerName"></param> /// <param name="log"></param> /// <param name="svcProvider"></param> public void Init(IProviderConfiguration providerConfig, string providerName, Logger log, IServiceProvider svcProvider) { if (providerConfig == null) { throw new ArgumentNullException("providerConfig"); } if (string.IsNullOrWhiteSpace(providerName)) { throw new ArgumentNullException("providerName"); } if (log == null) { throw new ArgumentNullException("log"); } if (svcProvider == null) { throw new ArgumentNullException("svcProvider"); } this.providerConfig = providerConfig; logger = log; serviceProvider = svcProvider; receivers = new ConcurrentDictionary <QueueId, EventHubAdapterReceiver>(); adapterConfig = new EventHubStreamProviderConfig(providerName); adapterConfig.PopulateFromProviderConfig(providerConfig); hubSettings = adapterConfig.GetEventHubSettings(providerConfig, serviceProvider); client = EventHubClient.CreateFromConnectionString(hubSettings.ConnectionString, hubSettings.Path); if (CheckpointerFactory == null) { checkpointerSettings = adapterConfig.GetCheckpointerSettings(providerConfig, serviceProvider); CheckpointerFactory = partition => EventHubCheckpointer.Create(checkpointerSettings, adapterConfig.StreamProviderName, partition); } if (CacheFactory == null) { var bufferPool = new FixedSizeObjectPool <FixedSizeBuffer>(adapterConfig.CacheSizeMb, pool => new FixedSizeBuffer(1 << 20, pool)); CacheFactory = (partition, checkpointer) => new EventHubQueueCache(checkpointer, bufferPool); } if (StreamFailureHandlerFactory == null) { //TODO: Add a queue specific default failure handler with reasonable error reporting. StreamFailureHandlerFactory = partition => Task.FromResult <IStreamFailureHandler>(new NoOpStreamDeliveryFailureHandler()); } if (QueueMapperFactory == null) { QueueMapperFactory = partitions => new EventHubQueueMapper(partitionIds, adapterConfig.StreamProviderName); } }
public void Alloc1Free1Test() { var accumulator = new Accumulator(); IObjectPool <TestPooledResource> pool = new FixedSizeObjectPool <TestPooledResource>(10, () => new TestPooledResource(accumulator)); // Allocate and free 20 items for (int i = 0; i < 20; i++) { TestPooledResource resource = pool.Allocate(); resource.Dispose(); } // only 1 at a time was ever allocated, so max allocated should be 1 Assert.Equal(1, accumulator.MaxAllocated); }
public void Alloc50Max10DoubleFreeTest() { var accumulator = new Accumulator(); IObjectPool <TestPooledResource> pool = new FixedSizeObjectPool <TestPooledResource>(10, () => new TestPooledResource(accumulator)); // Allocate 50 items var resources = Enumerable.Range(0, 50).Select(i => pool.Allocate()).ToList(); // pool was set to max of 10, so max allocated should be 10 Assert.Equal(10, accumulator.MaxAllocated); // free 50, note that 40 of these should already be freeded resources.ForEach(r => r.Dispose()); Assert.Equal(10, accumulator.MaxAllocated); }
private IEventHubQueueCache CreateQueueCache(string partition, IStreamQueueCheckpointer <string> checkpointer, Logger cacheLogger) { // the same code block as with EventHubAdapterFactory to define default CacheFactory // except for at the end we put the created cache in CreatedCaches list if (this.bufferPool == null) { this.bufferPool = new FixedSizeObjectPool <FixedSizeBuffer>(adapterSettings.CacheSizeMb, () => new FixedSizeBuffer(1 << 20)); } if (this.timePurge == null) { this.timePurge = new TimePurgePredicate(adapterSettings.DataMinTimeInCache, adapterSettings.DataMaxAgeInCache); } //ser defaultMaxAddCount to 10 so TryCalculateCachePressureContribution will start to calculate real contribution shortly. var cache = new EventHubQueueCache(defaultMaxAddCount, checkpointer, new EventHubDataAdapter(this.SerializationManager, bufferPool, timePurge), EventHubDataComparer.Instance, cacheLogger); if (adapterSettings.AveragingCachePressureMonitorFlowControlThreshold.HasValue) { var avgMonitor = new AveragingCachePressureMonitor(adapterSettings.AveragingCachePressureMonitorFlowControlThreshold.Value, cacheLogger); cache.AddCachePressureMonitor(avgMonitor); } if (adapterSettings.SlowConsumingMonitorPressureWindowSize.HasValue || adapterSettings.SlowConsumingMonitorFlowControlThreshold.HasValue) { var slowConsumeMonitor = new SlowConsumingPressureMonitor(cacheLogger); if (adapterSettings.SlowConsumingMonitorFlowControlThreshold.HasValue) { slowConsumeMonitor.FlowControlThreshold = adapterSettings.SlowConsumingMonitorFlowControlThreshold.Value; } if (adapterSettings.SlowConsumingMonitorPressureWindowSize.HasValue) { slowConsumeMonitor.PressureWindowSize = adapterSettings.SlowConsumingMonitorPressureWindowSize.Value; } cache.AddCachePressureMonitor(slowConsumeMonitor); } this.createdCaches.Add(cache); return(cache); }