public void Alloc10Free1Test()
{
var accumulator = new Accumulator();
IObjectPool <TestPooledResource> pool = new FixedSizeObjectPool <TestPooledResource>(10, p => new TestPooledResource(p, accumulator));
// Allocate 10 items
var resources = Enumerable.Range(0, 10).Select(i => pool.Allocate()).ToList();
// free 10
resources.ForEach(r => r.Dispose());
// pool was pre-populated with 10, so max allocated should be 10
Assert.AreEqual(10, accumulator.MaxAllocated);
Assert.AreEqual(0, accumulator.CurrentlyAllocated);
// Allocate and free 10 items
for (int i = 0; i < 10; i++)
{
TestPooledResource resource = pool.Allocate();
resource.Dispose();
}
// only 1 at a time was ever allocated, but pool was pre-populated with 10, so max allocated should be 10
Assert.AreEqual(10, accumulator.MaxAllocated);
Assert.AreEqual(0, accumulator.CurrentlyAllocated);
}
public EHPurgeLogicTests()
{
//an mock eh settings
this.ehSettings = new EventHubPartitionSettings();
ehSettings.Hub = new EventHubSettings();
ehSettings.Partition = "MockPartition";
//set up cache pressure monitor and purge predicate
this.cachePressureInjectionMonitor = new CachePressureInjectionMonitor();
this.purgePredicate = new PurgeDecisionInjectionPredicate(TimeSpan.FromMinutes(5), TimeSpan.FromMinutes(30));
//set up serialization env
var environment = SerializationTestEnvironment.InitializeWithDefaults();
this.serializationManager = environment.SerializationManager;
//set up buffer pool, small buffer size make it easy for cache to allocate multiple buffers
this.bufferPoolSizeInMB = EventHubStreamProviderSettings.DefaultCacheSizeMb;
var oneKB = 1024;
this.bufferPool = new FixedSizeObjectPool <FixedSizeBuffer>(this.bufferPoolSizeInMB, () => new FixedSizeBuffer(oneKB));
//set up logger
this.logger = new NoOpTestLogger().GetLogger(this.GetType().Name);
}
private IEventHubQueueCache CreateQueueCache(string partition, IStreamQueueCheckpointer <string> checkpointer)
{
var bufferPool = new FixedSizeObjectPool <FixedSizeBuffer>(adapterConfig.CacheSizeMb, pool => new FixedSizeBuffer(1 << 20, pool));
var dataAdapter = new CachedDataAdapter(partition, bufferPool);
return(new EventHubQueueCache(checkpointer, dataAdapter));
}
public IEventHubQueueCache CreateCache(string partition, IStreamQueueCheckpointer <string> checkpointer, Logger cacheLogger)
{
var bufferPool = new FixedSizeObjectPool <FixedSizeBuffer>(adapterSettings.CacheSizeMb, () => new FixedSizeBuffer(1 << 20));
var dataAdapter = new CachedDataAdapter(partition, bufferPool, timePurgePredicate, this.serializationManager);
return(new EventHubQueueCache(checkpointer, dataAdapter, EventHubDataComparer.Instance, cacheLogger));
}
private IEventHubQueueCache CreateCacheFactory(string partition, IStreamQueueCheckpointer <string> checkpointer, Logger cacheLogger,
FixedSizeObjectPool <FixedSizeBuffer> bufferPool, TimePurgePredicate timePurge)
{
var cache = new EventHubQueueCache(checkpointer, bufferPool, timePurge, cacheLogger, this.SerializationManager);
if (adapterSettings.AveragingCachePressureMonitorFlowControlThreshold.HasValue)
{
var avgMonitor = new AveragingCachePressureMonitor(adapterSettings.AveragingCachePressureMonitorFlowControlThreshold.Value, cacheLogger);
cache.AddCachePressureMonitor(avgMonitor);
}
if (adapterSettings.SlowConsumingMonitorPressureWindowSize.HasValue ||
adapterSettings.SlowConsumingMonitorFlowControlThreshold.HasValue)
{
var slowConsumeMonitor = new SlowConsumingPressureMonitor(cacheLogger);
if (adapterSettings.SlowConsumingMonitorFlowControlThreshold.HasValue)
{
slowConsumeMonitor.FlowControlThreshold = adapterSettings.SlowConsumingMonitorFlowControlThreshold.Value;
}
if (adapterSettings.SlowConsumingMonitorPressureWindowSize.HasValue)
{
slowConsumeMonitor.PressureWindowSize = adapterSettings.SlowConsumingMonitorPressureWindowSize.Value;
}
cache.AddCachePressureMonitor(slowConsumeMonitor);
}
return(cache);
}
/// <summary>
/// Factory initialization.
/// Provider config must contain the event hub settings type or the settings themselves.
/// EventHubSettingsType is recommended for consumers that do not want to include secure information in the cluster configuration.
/// </summary>
/// <param name="providerCfg"></param>
/// <param name="providerName"></param>
/// <param name="log"></param>
/// <param name="svcProvider"></param>
public virtual void Init(IProviderConfiguration providerCfg, string providerName, Logger log, IServiceProvider svcProvider)
{
if (providerCfg == null)
{
throw new ArgumentNullException(nameof(providerCfg));
}
if (string.IsNullOrWhiteSpace(providerName))
{
throw new ArgumentNullException(nameof(providerName));
}
if (log == null)
{
throw new ArgumentNullException(nameof(log));
}
providerConfig = providerCfg;
serviceProvider = svcProvider;
receivers = new ConcurrentDictionary <QueueId, EventHubAdapterReceiver>();
adapterSettings = new EventHubStreamProviderSettings(providerName);
adapterSettings.PopulateFromProviderConfig(providerConfig);
hubSettings = adapterSettings.GetEventHubSettings(providerConfig, serviceProvider);
client = EventHubClient.CreateFromConnectionString(hubSettings.ConnectionString, hubSettings.Path);
if (CheckpointerFactory == null)
{
checkpointerSettings = adapterSettings.GetCheckpointerSettings(providerConfig, serviceProvider);
CheckpointerFactory = partition => EventHubCheckpointer.Create(checkpointerSettings, adapterSettings.StreamProviderName, partition);
}
if (CacheFactory == null)
{
var bufferPool = new FixedSizeObjectPool <FixedSizeBuffer>(adapterSettings.CacheSizeMb, () => new FixedSizeBuffer(1 << 20));
var timePurge = new TimePurgePredicate(adapterSettings.DataMinTimeInCache, adapterSettings.DataMaxAgeInCache);
CacheFactory = (partition, checkpointer, cacheLogger) => new EventHubQueueCache(checkpointer, bufferPool, timePurge, cacheLogger);
}
if (StreamFailureHandlerFactory == null)
{
//TODO: Add a queue specific default failure handler with reasonable error reporting.
StreamFailureHandlerFactory = partition => Task.FromResult <IStreamFailureHandler>(new NoOpStreamDeliveryFailureHandler());
}
if (QueueMapperFactory == null)
{
QueueMapperFactory = partitions => new EventHubQueueMapper(partitionIds, adapterSettings.StreamProviderName);
}
if (ReceiverMonitorFactory == null)
{
ReceiverMonitorFactory = (hubPath, partition, receiverLogger) => new DefaultEventHubReceiverMonitor(hubPath, partition, receiverLogger.GetSubLogger("monitor", "-"));
}
logger = log.GetLogger($"EventHub.{hubSettings.Path}");
}
private IEventHubQueueCache CreateQueueCache(string partition, IStreamQueueCheckpointer <string> checkpointer, Logger log)
{
if (timePurgePredicate != null)
{
timePurgePredicate = new TimePurgePredicate(adapterConfig.DataMinTimeInCache, adapterConfig.DataMaxAgeInCache);
}
var bufferPool = new FixedSizeObjectPool <FixedSizeBuffer>(adapterConfig.CacheSizeMb, () => new FixedSizeBuffer(1 << 20));
var dataAdapter = new CachedDataAdapter(partition, bufferPool, timePurgePredicate);
return(new EventHubQueueCache(checkpointer, dataAdapter, log));
}
private IEventHubQueueCache CreateQueueCache(string partition, IStreamQueueCheckpointer <string> checkpointer, Logger log)
{
if (timePurgePredicate != null)
{
timePurgePredicate = new TimePurgePredicate(adapterSettings.DataMinTimeInCache, adapterSettings.DataMaxAgeInCache);
}
var bufferPool = new FixedSizeObjectPool <FixedSizeBuffer>(adapterSettings.CacheSizeMb, () => new FixedSizeBuffer(1 << 20));
var dataAdapter = new CachedDataAdapter(partition, bufferPool, timePurgePredicate, this.SerializationManager);
return(new EventHubQueueCache(checkpointer, dataAdapter, EventHubDataComparer.Instance, log));
}
public void Alloc50Max10Test()
{
var accumulator = new Accumulator();
IObjectPool <TestPooledResource> pool = new FixedSizeObjectPool <TestPooledResource>(10, () => new TestPooledResource(accumulator));
// Allocate 50 items
var resources = Enumerable.Range(0, 50).Select(i => pool.Allocate()).ToList();
// pool was set to max of 10, so max allocated should be 10
Assert.Equal(10, accumulator.MaxAllocated);
}
/// <summary>
/// Factory initialization.
/// Provider config must contain the event hub settings type or the settings themselves.
/// EventHubSettingsType is recommended for consumers that do not want to include secure information in the cluster configuration.
/// </summary>
/// <param name="providerConfig"></param>
/// <param name="providerName"></param>
/// <param name="log"></param>
/// <param name="svcProvider"></param>
public void Init(IProviderConfiguration providerConfig, string providerName, Logger log, IServiceProvider svcProvider)
{
if (providerConfig == null)
{
throw new ArgumentNullException("providerConfig");
}
if (string.IsNullOrWhiteSpace(providerName))
{
throw new ArgumentNullException("providerName");
}
if (log == null)
{
throw new ArgumentNullException("log");
}
if (svcProvider == null)
{
throw new ArgumentNullException("svcProvider");
}
this.providerConfig = providerConfig;
logger = log;
serviceProvider = svcProvider;
receivers = new ConcurrentDictionary <QueueId, EventHubAdapterReceiver>();
adapterConfig = new EventHubStreamProviderConfig(providerName);
adapterConfig.PopulateFromProviderConfig(providerConfig);
hubSettings = adapterConfig.GetEventHubSettings(providerConfig, serviceProvider);
client = EventHubClient.CreateFromConnectionString(hubSettings.ConnectionString, hubSettings.Path);
if (CheckpointerFactory == null)
{
checkpointerSettings = adapterConfig.GetCheckpointerSettings(providerConfig, serviceProvider);
CheckpointerFactory = partition => EventHubCheckpointer.Create(checkpointerSettings, adapterConfig.StreamProviderName, partition);
}
if (CacheFactory == null)
{
var bufferPool = new FixedSizeObjectPool <FixedSizeBuffer>(adapterConfig.CacheSizeMb, pool => new FixedSizeBuffer(1 << 20, pool));
CacheFactory = (partition, checkpointer) => new EventHubQueueCache(checkpointer, bufferPool);
}
if (StreamFailureHandlerFactory == null)
{
//TODO: Add a queue specific default failure handler with reasonable error reporting.
StreamFailureHandlerFactory = partition => Task.FromResult <IStreamFailureHandler>(new NoOpStreamDeliveryFailureHandler());
}
if (QueueMapperFactory == null)
{
QueueMapperFactory = partitions => new EventHubQueueMapper(partitionIds, adapterConfig.StreamProviderName);
}
}
public void Alloc1Free1Test()
{
var accumulator = new Accumulator();
IObjectPool <TestPooledResource> pool = new FixedSizeObjectPool <TestPooledResource>(10, () => new TestPooledResource(accumulator));
// Allocate and free 20 items
for (int i = 0; i < 20; i++)
{
TestPooledResource resource = pool.Allocate();
resource.Dispose();
}
// only 1 at a time was ever allocated, so max allocated should be 1
Assert.Equal(1, accumulator.MaxAllocated);
}
public void Alloc50Max10DoubleFreeTest()
{
var accumulator = new Accumulator();
IObjectPool <TestPooledResource> pool = new FixedSizeObjectPool <TestPooledResource>(10, () => new TestPooledResource(accumulator));
// Allocate 50 items
var resources = Enumerable.Range(0, 50).Select(i => pool.Allocate()).ToList();
// pool was set to max of 10, so max allocated should be 10
Assert.Equal(10, accumulator.MaxAllocated);
// free 50, note that 40 of these should already be freeded
resources.ForEach(r => r.Dispose());
Assert.Equal(10, accumulator.MaxAllocated);
}
private IEventHubQueueCache CreateQueueCache(string partition, IStreamQueueCheckpointer <string> checkpointer, Logger cacheLogger)
{
// the same code block as with EventHubAdapterFactory to define default CacheFactory
// except for at the end we put the created cache in CreatedCaches list
if (this.bufferPool == null)
{
this.bufferPool = new FixedSizeObjectPool <FixedSizeBuffer>(adapterSettings.CacheSizeMb, () => new FixedSizeBuffer(1 << 20));
}
if (this.timePurge == null)
{
this.timePurge = new TimePurgePredicate(adapterSettings.DataMinTimeInCache, adapterSettings.DataMaxAgeInCache);
}
//ser defaultMaxAddCount to 10 so TryCalculateCachePressureContribution will start to calculate real contribution shortly.
var cache = new EventHubQueueCache(defaultMaxAddCount, checkpointer, new EventHubDataAdapter(this.SerializationManager, bufferPool, timePurge), EventHubDataComparer.Instance, cacheLogger);
if (adapterSettings.AveragingCachePressureMonitorFlowControlThreshold.HasValue)
{
var avgMonitor = new AveragingCachePressureMonitor(adapterSettings.AveragingCachePressureMonitorFlowControlThreshold.Value, cacheLogger);
cache.AddCachePressureMonitor(avgMonitor);
}
if (adapterSettings.SlowConsumingMonitorPressureWindowSize.HasValue ||
adapterSettings.SlowConsumingMonitorFlowControlThreshold.HasValue)
{
var slowConsumeMonitor = new SlowConsumingPressureMonitor(cacheLogger);
if (adapterSettings.SlowConsumingMonitorFlowControlThreshold.HasValue)
{
slowConsumeMonitor.FlowControlThreshold = adapterSettings.SlowConsumingMonitorFlowControlThreshold.Value;
}
if (adapterSettings.SlowConsumingMonitorPressureWindowSize.HasValue)
{
slowConsumeMonitor.PressureWindowSize = adapterSettings.SlowConsumingMonitorPressureWindowSize.Value;
}
cache.AddCachePressureMonitor(slowConsumeMonitor);
}
this.createdCaches.Add(cache);
return(cache);
}
