/// <inheritdoc/>
public override Task Initialize(string strProviderName, IStreamQueueMapper queueMapper, TimeSpan siloMaturityPeriod)
{
if (queueMapper == null)
{
throw new ArgumentNullException("queueMapper");
}
var options = this.serviceProvider.GetRequiredService <IOptionsSnapshot <LeaseBasedQueueBalancerOptions> >().Get(strProviderName);
if (options == null)
{
throw new KeyNotFoundException($"No lease base queue balancer options was configured for provider {strProviderName}, nor was a default configured.");
}
this.leaseProvider = this.serviceProvider.GetRequiredService(options.LeaseProviderType) as ILeaseProvider;
this.leaseLength = options.LeaseLength;
this.allQueues = new ReadOnlyCollection <QueueId>(queueMapper.GetAllQueues().ToList());
this.siloMaturityPeriod = siloMaturityPeriod;
NotifyAfterStart().Ignore();
//make lease renew frequency to be every half of lease time, to avoid renew failing due to timing issues, race condition or clock difference.
var timerLogger = this.loggerFactory.CreateLogger <AsyncTaskSafeTimer>();
this.renewLeaseTimer = new AsyncTaskSafeTimer(timerLogger, this.MaintainAndBalanceQueues, null, this.siloMaturityPeriod, this.leaseLength.Divide(2));
//try to acquire maximum leases every leaseLength
this.tryAcquireMaximumLeaseTimer = new AsyncTaskSafeTimer(timerLogger, this.AcquireLeaseToMeetMaxResponsibilty, null, this.siloMaturityPeriod, this.leaseLength);
//Selector default to round robin selector now, but we can make a further change to make selector configurable if needed. Selector algorithm could
//be affecting queue balancing stablization time in cluster initializing and auto-scaling
this.queueSelector = new RoundRobinSelector <QueueId>(this.allQueues);
return(MaintainAndBalanceQueues(null));
}
/// <summary>
/// Create stream queue balancer by type requested
/// </summary>
/// <param name="balancerType">queue balancer type to create</param>
/// <param name="strProviderName">name of requesting stream provider</param>
/// <param name="runtime">stream provider runtime environment to run in</param>
/// <param name="queueMapper">queue mapper of requesting stream provider</param>
/// <returns>Constructed stream queue balancer</returns>
public static IStreamQueueBalancer Create(
StreamQueueBalancerType balancerType,
string strProviderName,
IStreamProviderRuntime runtime,
IStreamQueueMapper queueMapper)
{
if (string.IsNullOrWhiteSpace(strProviderName))
{
throw new ArgumentNullException("strProviderName");
}
if (runtime == null)
{
throw new ArgumentNullException("runtime");
}
if (queueMapper == null)
{
throw new ArgumentNullException("queueMapper");
}
switch (balancerType)
{
case StreamQueueBalancerType.ConsistenRingBalancer:
{
// Consider: for now re-use the same ConsistentRingProvider with 1 equally devided range. Remove later.
IConsistentRingProviderForGrains ringProvider = runtime.GetConsistentRingProvider(0, 1);
return(new ConsistentRingQueueBalancer(ringProvider, queueMapper));
}
default:
{
string error = string.Format("Unsupported balancerType for stream provider. BalancerType: {0}, StreamProvider: {1}", balancerType, strProviderName);
throw new ArgumentOutOfRangeException("balancerType", error);
}
}
}
public virtual void Init()
{
this.receivers = new ConcurrentDictionary <QueueId, KafkaEventBusAdapterReceiver>();
this.telemetryProducer = this.serviceProvider.GetService <ITelemetryProducer>();
if (producer == null)
{
InitKafkaProducer();
}
_adapterCache = new SimpleQueueAdapterCache(new SimpleQueueCacheOptions {
CacheSize = cacheOptions.CacheSize
}, Name, loggerFactory);
if (this.StreamFailureHandlerFactory == null)
{
//TODO: Add a queue specific default failure handler with reasonable error reporting.
this.StreamFailureHandlerFactory = partition => Task.FromResult <IStreamFailureHandler>(new NoOpStreamDeliveryFailureHandler());
}
if (this.QueueMapperFactory == null)
{
var h = new HashRingStreamQueueMapperOptions {
TotalQueueCount = receiverOptions.TotalQueueCount
};
this.streamQueueMapper = new HashRingBasedStreamQueueMapper(h, Name);
this.QueueMapperFactory = () => new HashRingBasedStreamQueueMapper(h, this.Name);
}
this.logger = this.loggerFactory.CreateLogger($"{this.GetType().FullName}"); // join topics?
}
/// <summary>
/// Create stream queue balancer by type requested
/// </summary>
/// <param name="balancerType">queue balancer type to create</param>
/// <param name="strProviderName">name of requesting stream provider</param>
/// <param name="siloStatusOracle">membership services interface.</param>
/// <param name="clusterConfiguration">cluster configuration</param>
/// <param name="runtime">stream provider runtime environment to run in</param>
/// <param name="queueMapper">queue mapper of requesting stream provider</param>
/// <param name="siloMaturityPeriod">Maturity Period of a silo for queue rebalancing purposes</param>
/// <returns>Constructed stream queue balancer</returns>
public static IStreamQueueBalancer Create(
StreamQueueBalancerType balancerType,
string strProviderName,
ISiloStatusOracle siloStatusOracle,
ClusterConfiguration clusterConfiguration,
IStreamProviderRuntime runtime,
IStreamQueueMapper queueMapper,
TimeSpan siloMaturityPeriod)
{
if (string.IsNullOrWhiteSpace(strProviderName))
{
throw new ArgumentNullException("strProviderName");
}
if (siloStatusOracle == null)
{
throw new ArgumentNullException("siloStatusOracle");
}
if (clusterConfiguration == null)
{
throw new ArgumentNullException("clusterConfiguration");
}
if (runtime == null)
{
throw new ArgumentNullException("runtime");
}
if (queueMapper == null)
{
throw new ArgumentNullException("queueMapper");
}
bool isFixed;
switch (balancerType)
{
case StreamQueueBalancerType.ConsistentRingBalancer:
{
// Consider: for now re-use the same ConsistentRingProvider with 1 equally devided range. Remove later.
IConsistentRingProviderForGrains ringProvider = runtime.GetConsistentRingProvider(0, 1);
return new ConsistentRingQueueBalancer(ringProvider, queueMapper);
}
case StreamQueueBalancerType.DynamicAzureDeploymentBalancer:
case StreamQueueBalancerType.StaticAzureDeploymentBalancer:
{
Logger logger = LogManager.GetLogger(typeof(StreamQueueBalancerFactory).Name, LoggerType.Runtime);
var wrapper = AssemblyLoader.LoadAndCreateInstance<IDeploymentConfiguration>(Constants.ORLEANS_AZURE_UTILS_DLL, logger);
isFixed = balancerType == StreamQueueBalancerType.StaticAzureDeploymentBalancer;
return new DeploymentBasedQueueBalancer(siloStatusOracle, wrapper, queueMapper, siloMaturityPeriod, isFixed);
}
case StreamQueueBalancerType.DynamicClusterConfigDeploymentBalancer:
case StreamQueueBalancerType.StaticClusterConfigDeploymentBalancer:
{
IDeploymentConfiguration deploymentConfiguration = new StaticClusterDeploymentConfiguration(clusterConfiguration);
isFixed = balancerType == StreamQueueBalancerType.StaticClusterConfigDeploymentBalancer;
return new DeploymentBasedQueueBalancer(siloStatusOracle, deploymentConfiguration, queueMapper, siloMaturityPeriod, isFixed);
}
default:
{
string error = string.Format("Unsupported balancerType for stream provider. BalancerType: {0}, StreamProvider: {1}", balancerType, strProviderName);
throw new ArgumentOutOfRangeException("balancerType", error);
}
}
}
/// <summary>
/// Factory initialization.
/// </summary>
/// <param name="providerConfig"></param>
/// <param name="name"></param>
/// <param name="svcProvider"></param>
public void Init(IProviderConfiguration providerConfig, string name, IServiceProvider svcProvider)
{
logger = svcProvider.GetService <ILogger <MemoryAdapterFactory <TSerializer> > >();
this.loggerFactory = svcProvider.GetRequiredService <ILoggerFactory>();
serviceProvider = svcProvider;
providerName = name;
queueGrains = new ConcurrentDictionary <QueueId, IMemoryStreamQueueGrain>();
adapterConfig = new MemoryAdapterConfig(providerName);
this.telemetryProducer = svcProvider.GetService <ITelemetryProducer>();
if (CacheMonitorFactory == null)
{
this.CacheMonitorFactory = (dimensions, telemetryProducer) => new DefaultCacheMonitor(dimensions, telemetryProducer);
}
if (this.BlockPoolMonitorFactory == null)
{
this.BlockPoolMonitorFactory = (dimensions, telemetryProducer) => new DefaultBlockPoolMonitor(dimensions, telemetryProducer);
}
if (this.ReceiverMonitorFactory == null)
{
this.ReceiverMonitorFactory = (dimensions, telemetryProducer) => new DefaultQueueAdapterReceiverMonitor(dimensions, telemetryProducer);
}
purgePredicate = new TimePurgePredicate(adapterConfig.DataMinTimeInCache, adapterConfig.DataMaxAgeInCache);
grainFactory = (IGrainFactory)serviceProvider.GetService(typeof(IGrainFactory));
adapterConfig.PopulateFromProviderConfig(providerConfig);
streamQueueMapper = new HashRingBasedStreamQueueMapper(adapterConfig.TotalQueueCount, adapterConfig.StreamProviderName);
this.sharedDimensions = new MonitorAggregationDimensions(serviceProvider.GetService <GlobalConfiguration>(), serviceProvider.GetService <NodeConfiguration>());
this.serializer = MemoryMessageBodySerializerFactory <TSerializer> .GetOrCreateSerializer(svcProvider);
}
public KafkaAdapterFactory(
string name,
KafkaStreamOptions options,
SimpleQueueCacheOptions cacheOptions,
SerializationManager serializationManager,
ILoggerFactory loggerFactory,
IGrainFactory grainFactory
)
{
_options = options ?? throw new ArgumentNullException(nameof(options));
_name = name;
_serializationManager = serializationManager;
_loggerFactory = loggerFactory;
_grainFactory = grainFactory;
_logger = loggerFactory.CreateLogger <KafkaAdapterFactory>();
if (options.Topics != null && options.Topics.Count == 0)
{
throw new ArgumentNullException(nameof(options.Topics));
}
_adapterCache = new SimpleQueueAdapterCache(
cacheOptions,
name,
loggerFactory
);
_queueProperties = GetQueuesProperties();
_streamQueueMapper = new ExternalQueueMapper(_queueProperties.Values);
}
public DeploymentBasedQueueBalancer(
ISiloStatusOracle siloStatusOracle,
IDeploymentConfiguration deploymentConfig,
IStreamQueueMapper queueMapper,
bool isFixed)
{
if (siloStatusOracle == null)
{
throw new ArgumentNullException("siloStatusOracle");
}
if (deploymentConfig == null)
{
throw new ArgumentNullException("deploymentConfig");
}
if (queueMapper == null)
{
throw new ArgumentNullException("queueMapper");
}
this.siloStatusOracle = siloStatusOracle;
this.deploymentConfig = deploymentConfig;
allQueues = queueMapper.GetAllQueues().ToList();
queueBalanceListeners = new List<IStreamQueueBalanceListener>();
mySiloName = this.siloStatusOracle.SiloName;
this.isFixed = isFixed;
// register for notification of changes to silo status for any silo in the cluster
this.siloStatusOracle.SubscribeToSiloStatusEvents(this);
}
public RabbitMQAdapterFactory(
string name,
RabbitMQStreamProviderOptions config,
HashRingStreamQueueMapperOptions queueMapperOptions,
SimpleQueueCacheOptions cacheOptions,
IServiceProvider serviceProvider,
IOptions <ClusterOptions> clusterOptions,
SerializationManager serializationManager,
ILoggerFactory loggerFactory,
IMessageSerializationHandler serializationHandler,
IRabbitMQMapper mapper)
{
_config = config;
_providerName = name;
_loggeFactory = loggerFactory;
_serializationHandler = serializationHandler;
_mapper = mapper;
_mapper.Init( );
_cacheSize = cacheOptions.CacheSize;
_adapterCache = new SimpleQueueAdapterCache(cacheOptions, _providerName, _loggeFactory);
_streamQueueMapper = new HashRingBasedStreamQueueMapper(queueMapperOptions, _providerName);
}
public RabbitMqAdapter(RabbitMqStreamProviderOptions options, SerializationManager serializationManager, IStreamQueueMapper mapper, string providerName, Logger logger)
{
_serializationManager = serializationManager;
_mapper = mapper;
Name = providerName;
_rmqConnectorFactory = new RabbitMqOnlineConnectorFactory(options, logger);
_cacheFillingTimeout = options.CacheFillingTimeout;
}
public RabbitMqAdapter(RabbitMqOptions rmqOptions, CachingOptions cachingOptions, IBatchContainerSerializer serializer, IStreamQueueMapper mapper, string providerName, ILoggerFactory loggerFactory)
{
_serializer = serializer;
_mapper = mapper;
Name = providerName;
_rmqConnectorFactory = new RabbitMqOnlineConnectorFactory(rmqOptions, loggerFactory);
_cacheFillingTimeout = cachingOptions.CacheFillingTimeout;
}
public Task <IQueueAdapter> CreateAdapter()
{
if (this.streamQueueMapper == null)
{
this.streamQueueMapper = this.QueueMapperFactory();
}
return(Task.FromResult(this as IQueueAdapter));
}
public RabbitMqAdapter(RabbitMqOptions rmqOptions, IQueueDataAdapter <RabbitMqMessage, IEnumerable <IBatchContainer> > dataAdapter, string providerName, IStreamQueueMapper mapper, IRabbitMqConnectorFactory rmqConnectorFactory)
{
_dataAdapter = dataAdapter;
Name = providerName;
_mapper = mapper;
_rmqConnectorFactory = rmqConnectorFactory;
_rmqOptions = rmqOptions;
_producer = new ThreadLocal <IRabbitMqProducer>(() => _rmqConnectorFactory.CreateProducer());
}
public async Task Initialize(string strProviderName,
IStreamQueueMapper queueMapper)
{
this.leaseManagerGrain = this.grainFactory.GetGrain <ILeaseManagerGrain>(strProviderName);
await this.leaseManagerGrain.SetQueuesAsLeases(queueMapper.GetAllQueues());
this.id = $"{strProviderName}-{Guid.NewGuid()}";
await GetInitialLease();
}
public RabbitMQAdapter(RabbitMQStreamProviderConfig config, Logger logger, string providerName, IStreamQueueMapper streamQueueMapper, IRabbitMQMapper mapper)
{
_config = config;
_logger = logger;
Name = providerName;
_streamQueueMapper = streamQueueMapper;
_mapper = mapper;
_queues = new ConcurrentDictionary <QueueId, object>();
}
/// <summary>
/// Create stream queue balancer by type requested
/// </summary>
/// <param name="balancerType">queue balancer type to create</param>
/// <param name="strProviderName">name of requesting stream provider</param>
/// <param name="siloStatusOracle">membership services interface.</param>
/// <param name="clusterConfiguration">cluster configuration</param>
/// <param name="runtime">stream provider runtime environment to run in</param>
/// <param name="queueMapper">queue mapper of requesting stream provider</param>
/// <returns>Constructed stream queue balancer</returns>
public static IStreamQueueBalancer Create(
StreamQueueBalancerType balancerType,
string strProviderName,
ISiloStatusOracle siloStatusOracle,
ClusterConfiguration clusterConfiguration,
IStreamProviderRuntime runtime,
IStreamQueueMapper queueMapper)
{
if (string.IsNullOrWhiteSpace(strProviderName))
{
throw new ArgumentNullException("strProviderName");
}
if (siloStatusOracle == null)
{
throw new ArgumentNullException("siloStatusOracle");
}
if (clusterConfiguration == null)
{
throw new ArgumentNullException("clusterConfiguration");
}
if (runtime == null)
{
throw new ArgumentNullException("runtime");
}
if (queueMapper == null)
{
throw new ArgumentNullException("queueMapper");
}
switch (balancerType)
{
case StreamQueueBalancerType.ConsistentRingBalancer:
{
// Consider: for now re-use the same ConsistentRingProvider with 1 equally devided range. Remove later.
IConsistentRingProviderForGrains ringProvider = runtime.GetConsistentRingProvider(0, 1);
return(new ConsistentRingQueueBalancer(ringProvider, queueMapper));
}
case StreamQueueBalancerType.AzureDeploymentBalancer:
{
TraceLogger logger = TraceLogger.GetLogger(typeof(StreamQueueBalancerFactory).Name, TraceLogger.LoggerType.Runtime);
var wrapper = AssemblyLoader.LoadAndCreateInstance <IDeploymentConfiguration>(Constants.ORLEANS_AZURE_UTILS_DLL, logger);
return(new DeploymentBasedQueueBalancer(siloStatusOracle, wrapper, queueMapper));
}
case StreamQueueBalancerType.StaticClusterDeploymentBalancer:
{
IDeploymentConfiguration deploymentConfiguration = new StaticClusterDeploymentConfiguration(clusterConfiguration);
return(new DeploymentBasedQueueBalancer(siloStatusOracle, deploymentConfiguration, queueMapper));
}
default:
{
string error = string.Format("Unsupported balancerType for stream provider. BalancerType: {0}, StreamProvider: {1}", balancerType, strProviderName);
throw new ArgumentOutOfRangeException("balancerType", error);
}
}
}
public override Task Initialize(IStreamQueueMapper queueMapper)
{
if (queueMapper == null)
{
throw new ArgumentNullException("queueMapper");
}
this.allQueues = new ReadOnlyCollection <QueueId>(queueMapper.GetAllQueues().ToList());
NotifyAfterStart().Ignore();
return(Task.CompletedTask);
}
public GenericQueueAdapter(Logger logger, IStreamQueueMapper streamQueueMapper, string providerName, IProviderConfiguration config, IProviderQueue queueProvider, int numOfQueues)
{
Name = providerName; // WTF: If you set the name to anything else, the client won't receive any messages !?????
_config = config;
_logger = logger;
_streamQueueMapper = streamQueueMapper;
_queueProvider = queueProvider;
_providerName = providerName;
_numOfQueues = numOfQueues;
}
public override Task Initialize(IStreamQueueMapper queueMapper)
{
if (queueMapper == null)
{
throw new ArgumentNullException("queueMapper");
}
this.allQueues = queueMapper.GetAllQueues().ToList();
NotifyAfterStart().Ignore();
return(base.Initialize(queueMapper));
}
public RabbitMqAdapterReceiver(IRabbitMqConnectorFactory rmqConnectorFactory, QueueId queueId, IStreamQueueMapper mapper, IQueueDataAdapter <RabbitMqMessage, IEnumerable <IBatchContainer> > dataAdapter)
{
_rmqConnectorFactory = rmqConnectorFactory;
_queueId = queueId;
_mapper = mapper;
_dataAdapter = dataAdapter;
_logger = _rmqConnectorFactory.LoggerFactory.CreateLogger($"{typeof(RabbitMqAdapterReceiver).FullName}.{queueId}");
_sequenceId = 0;
pending = new List <PendingDelivery>();
_currentGroup = new Queue <IBatchContainer>();
}
public RabbitMQAdapter(RabbitMQStreamProviderOptions config, ILoggerFactory loggerFactory, string providerName, IStreamQueueMapper streamQueueMapper, IRabbitMQMapper mapper)
{
Direction = config.Mode;
_config = config;
_loggerFactory = loggerFactory;
Name = providerName;
_streamQueueMapper = streamQueueMapper;
_mapper = mapper;
_queues = new ConcurrentDictionary <QueueId, object>();
CreateConnection();
}
public async Task Initialize(string strProviderName,
IStreamQueueMapper queueMapper,
TimeSpan siloMaturityPeriod,
IProviderConfiguration providerConfig)
{
this.leaseManagerGrain = this.grainFactory.GetGrain <ILeaseManagerGrain>(strProviderName);
await this.leaseManagerGrain.SetQueuesAsLeases(queueMapper.GetAllQueues());
this.id = $"{strProviderName}-{Guid.NewGuid()}";
await GetInitialLease();
}
public RedisQueueAdapter(
RedisStreamOptions options,
IConnectionMultiplexerFactory connectionMultiplexerFactory,
IRedisDataAdapter dataAdapter,
IStreamQueueMapper streamQueueMapper,
ILogger logger,
string serviceId,
string clusterId,
string providerName)
{
if (options == null)
{
throw new ArgumentNullException(nameof(options));
}
if (connectionMultiplexerFactory == null)
{
throw new ArgumentNullException(nameof(connectionMultiplexerFactory));
}
if (dataAdapter == null)
{
throw new ArgumentNullException(nameof(dataAdapter));
}
if (streamQueueMapper == null)
{
throw new ArgumentNullException(nameof(streamQueueMapper));
}
if (string.IsNullOrEmpty(serviceId))
{
throw new ArgumentNullException(nameof(serviceId));
}
if (string.IsNullOrEmpty(clusterId))
{
throw new ArgumentNullException(nameof(clusterId));
}
if (string.IsNullOrEmpty(providerName))
{
throw new ArgumentNullException(nameof(providerName));
}
_redisStreamOptions = options;
_connectionMultiplexerFactory = connectionMultiplexerFactory;
ServiceId = serviceId;
ClusterId = clusterId;
Name = providerName;
_streamQueueMapper = streamQueueMapper;
_dataAdapter = dataAdapter;
_logger = logger.ForContext <RedisQueueAdapter>(new Dictionary <string, object>
{
{ "ServiceId", serviceId },
{ "ProviderName", providerName }
});
}
public override Task Initialize(string strProviderName,
IStreamQueueMapper queueMapper,
TimeSpan siloMaturityPeriod)
{
if (queueMapper == null)
{
throw new ArgumentNullException("queueMapper");
}
this.allQueues = new ReadOnlyCollection <QueueId>(queueMapper.GetAllQueues().ToList());
this.siloMaturityPeriod = siloMaturityPeriod;
NotifyAfterStart().Ignore();
return(Task.CompletedTask);
}
public override Task Initialize(IStreamQueueMapper queueMapper)
{
if (queueMapper == null)
{
throw new ArgumentNullException("queueMapper");
}
if (!(queueMapper is IConsistentRingStreamQueueMapper))
{
throw new ArgumentException("queueMapper for ConsistentRingQueueBalancer should implement IConsistentRingStreamQueueMapper", "queueMapper");
}
streamQueueMapper = (IConsistentRingStreamQueueMapper)queueMapper;
return(base.Initialize(queueMapper));
}
public void Init(IProviderConfiguration providerConfig, string providerName, Logger log, IServiceProvider svcProvider)
{
this.logger = log;
this.serviceProvider = svcProvider;
this.providerName = providerName;
this.queueGrains = new ConcurrentDictionary <QueueId, IMemoryStreamQueueGrain>();
this.adapterConfig = new MemoryAdapterConfig(providerName);
grainFactory = (GrainFactory)serviceProvider.GetService(typeof(GrainFactory));
adapterConfig.PopulateFromProviderConfig(providerConfig);
this.streamQueueMapper = new HashRingBasedStreamQueueMapper(adapterConfig.TotalQueueCount, adapterConfig.StreamProviderName);
// 10 meg buffer pool. 10 1 meg blocks
this.bufferPool = new FixedSizeObjectPool <FixedSizeBuffer>(adapterConfig.CacheSizeMb, () => new FixedSizeBuffer(1 << 20));
}
public override Task Initialize(string strProviderName,
IStreamQueueMapper queueMapper)
{
if (queueMapper == null)
{
throw new ArgumentNullException("queueMapper");
}
if (!(queueMapper is IConsistentRingStreamQueueMapper))
{
throw new ArgumentException("queueMapper for ConsistentRingQueueBalancer should implement IConsistentRingStreamQueueMapper", "queueMapper");
}
streamQueueMapper = (IConsistentRingStreamQueueMapper)queueMapper;
return(Task.CompletedTask);
}
/// <summary>
/// Factory initialization.
/// </summary>
/// <param name="providerConfig"></param>
/// <param name="name"></param>
/// <param name="log"></param>
/// <param name="svcProvider"></param>
public void Init(IProviderConfiguration providerConfig, string name, Logger log, IServiceProvider svcProvider)
{
logger = log;
serviceProvider = svcProvider;
providerName = name;
queueGrains = new ConcurrentDictionary <QueueId, IMemoryStreamQueueGrain>();
adapterConfig = new MemoryAdapterConfig(providerName);
grainFactory = (IGrainFactory)serviceProvider.GetService(typeof(IGrainFactory));
adapterConfig.PopulateFromProviderConfig(providerConfig);
streamQueueMapper = new HashRingBasedStreamQueueMapper(adapterConfig.TotalQueueCount, adapterConfig.StreamProviderName);
// 10 meg buffer pool. 10 1 meg blocks
bufferPool = new FixedSizeObjectPool <FixedSizeBuffer>(() => new FixedSizeBuffer(1 << 20), adapterConfig.CacheSizeMb);
this.serializer = MemoryMessageBodySerializerFactory <TSerializer> .GetOrCreateSerializer(svcProvider);
}
/// <inheritdoc/>
public override Task Initialize(IStreamQueueMapper queueMapper)
{
if (base.Cancellation.IsCancellationRequested)
{
throw new InvalidOperationException("Cannot initialize a terminated balancer.");
}
if (queueMapper == null)
{
throw new ArgumentNullException("queueMapper");
}
this.allQueues = new ReadOnlyCollection <QueueId>(queueMapper.GetAllQueues().ToList());
//Selector default to round robin selector now, but we can make a further change to make selector configurable if needed. Selector algorithm could
//be affecting queue balancing stablization time in cluster initializing and auto-scaling
this.queueSelector = new RoundRobinSelector <QueueId>(this.allQueues, new Random(this.GetHashCode()));
return(base.Initialize(queueMapper));
}
public EventStoreAdapterFactory(string providerName, IEventStoreStreamProviderConfiguration streamProviderConfiguration, ILoggerFactory loggerFactory)
{
_streamProviderConfiguration = streamProviderConfiguration;
_loggerFactory = loggerFactory;
_providerName = providerName;
_receivers = new ConcurrentDictionary <QueueId, EventStoreQueueAdapterReceiver>();
_eventStoreQueueAdapterCache = new EventStoreQueueAdapterCache(this, loggerFactory);
var hashRingStreamQueueMapperOptions = new HashRingStreamQueueMapperOptions()
{
TotalQueueCount = 2
};
_streamQueueMapper = new HashRingBasedStreamQueueMapper(hashRingStreamQueueMapperOptions, _providerName);
}
public RabbitMqAdapterFactory(
string providerName,
IOptionsMonitor <RabbitMqOptions> rmqOptionsAccessor,
IOptionsMonitor <SimpleQueueCacheOptions> cachingOptionsAccessor,
IServiceProvider serviceProvider,
ILoggerFactory loggerFactory,
IRabbitMqStreamQueueMapperFactory streamQueueMapperFactory)
{
if (string.IsNullOrEmpty(providerName))
{
throw new ArgumentNullException(nameof(providerName));
}
if (rmqOptionsAccessor == null)
{
throw new ArgumentNullException(nameof(rmqOptionsAccessor));
}
if (cachingOptionsAccessor == null)
{
throw new ArgumentNullException(nameof(cachingOptionsAccessor));
}
if (serviceProvider == null)
{
throw new ArgumentNullException(nameof(serviceProvider));
}
if (loggerFactory == null)
{
throw new ArgumentNullException(nameof(loggerFactory));
}
if (streamQueueMapperFactory == null)
{
throw new ArgumentNullException(nameof(streamQueueMapperFactory));
}
var rmqOptions = rmqOptionsAccessor.Get(providerName);
var cachingOptions = cachingOptionsAccessor.Get(providerName);
_cache = new SimpleQueueAdapterCache(cachingOptions, providerName, loggerFactory);
_mapper = streamQueueMapperFactory.Get(providerName);
_failureHandler = Task.FromResult <IStreamFailureHandler>(new NoOpStreamDeliveryFailureHandler(false));
var dataAdapter = serviceProvider.GetServiceByName <IQueueDataAdapter <RabbitMqMessage, IEnumerable <IBatchContainer> > >(providerName) ??
RabbitMqDataAdapter.Create(serviceProvider, providerName);
_adapter = new RabbitMqAdapter(rmqOptions, dataAdapter, providerName, _mapper, serviceProvider.GetRequiredServiceByName <IRabbitMqConnectorFactory>(providerName));
}
public TestAdapterFactory(
string name,
SimpleQueueCacheOptions cacheOptions,
Func <IEnumerable <IBatchContainer> > queueMessagesProvider,
Action <IEnumerable <IBatchContainer> > onMessagesDelivered,
ILoggerFactory loggerFactory
)
{
_providerName = name;
_queueMessagesProvider = queueMessagesProvider ?? throw new ArgumentNullException(nameof(queueMessagesProvider));
_onMessagesDelivered = onMessagesDelivered ?? throw new ArgumentNullException(nameof(onMessagesDelivered));
_loggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory));
_streamQueueMapper = new HashRingBasedStreamQueueMapper(new HashRingStreamQueueMapperOptions()
{
TotalQueueCount = 1
}, _providerName);
_adapterCache = new SimpleQueueAdapterCache(cacheOptions, _providerName, _loggerFactory);
}
/// <summary>
/// Factory initialization.
/// </summary>
public void Init()
{
this.queueGrains = new ConcurrentDictionary <QueueId, IMemoryStreamQueueGrain>();
if (CacheMonitorFactory == null)
{
this.CacheMonitorFactory = (dimensions, telemetryProducer) => new DefaultCacheMonitor(dimensions, telemetryProducer);
}
if (this.BlockPoolMonitorFactory == null)
{
this.BlockPoolMonitorFactory = (dimensions, telemetryProducer) => new DefaultBlockPoolMonitor(dimensions, telemetryProducer);
}
if (this.ReceiverMonitorFactory == null)
{
this.ReceiverMonitorFactory = (dimensions, telemetryProducer) => new DefaultQueueAdapterReceiverMonitor(dimensions, telemetryProducer);
}
this.purgePredicate = new TimePurgePredicate(this.cacheOptions.DataMinTimeInCache, this.cacheOptions.DataMaxAgeInCache);
this.streamQueueMapper = new HashRingBasedStreamQueueMapper(this.queueMapperOptions, this.Name);
}
public DeploymentBasedQueueBalancer(
ISiloStatusOracle siloStatusOracle,
IDeploymentConfiguration deploymentConfig,
IStreamQueueMapper queueMapper,
TimeSpan maturityPeriod,
bool isFixed)
{
if (siloStatusOracle == null)
{
throw new ArgumentNullException("siloStatusOracle");
}
if (deploymentConfig == null)
{
throw new ArgumentNullException("deploymentConfig");
}
if (queueMapper == null)
{
throw new ArgumentNullException("queueMapper");
}
this.siloStatusOracle = siloStatusOracle;
this.deploymentConfig = deploymentConfig;
allQueues = new ReadOnlyCollection<QueueId>(queueMapper.GetAllQueues().ToList());
queueBalanceListeners = new List<IStreamQueueBalanceListener>();
immatureSilos = new ConcurrentDictionary<SiloAddress, bool>();
this.isFixed = isFixed;
siloMaturityPeriod = maturityPeriod;
isStarting = true;
// register for notification of changes to silo status for any silo in the cluster
this.siloStatusOracle.SubscribeToSiloStatusEvents(this);
// record all already active silos as already mature.
// Even if they are not yet, they will be mature by the time I mature myself (after I become !isStarting).
foreach (var silo in siloStatusOracle.GetApproximateSiloStatuses(true).Keys.Where(s => !s.Equals(siloStatusOracle.SiloAddress)))
{
immatureSilos[silo] = false; // record as mature
}
NotifyAfterStart().Ignore();
}
public DeploymentBasedQueueBalancer(
ISiloStatusOracle siloStatusOracle,
IDeploymentConfiguration deploymentConfig,
IStreamQueueMapper queueMapper)
{
if (siloStatusOracle == null)
{
throw new ArgumentNullException("siloStatusOracle");
}
if (deploymentConfig == null)
{
throw new ArgumentNullException("deploymentConfig");
}
if (queueMapper == null)
{
throw new ArgumentNullException("queueMapper");
}
this.siloStatusOracle = siloStatusOracle;
this.deploymentConfig = deploymentConfig;
streamQueueMapper = queueMapper;
queueBalanceListeners = new List<IStreamQueueBalanceListener>();
mySiloName = this.siloStatusOracle.SiloName;
activeSiloNames = new List<string>();
allSiloNames = this.deploymentConfig.GetAllSiloInstanceNames();
List<QueueId> allQueues = streamQueueMapper.GetAllQueues().ToList();
resourceBalancer = new BestFitBalancer<string, QueueId>(allSiloNames, allQueues);
// get silo names for all active silos
foreach (SiloAddress siloAddress in this.siloStatusOracle.GetApproximateSiloStatuses(true).Keys)
{
string siloName;
if (this.siloStatusOracle.TryGetSiloName(siloAddress, out siloName))
{
activeSiloNames.Add(siloName);
}
}
// register for notification of changes to silo status for any silo in the cluster
this.siloStatusOracle.SubscribeToSiloStatusEvents(this);
}
public ConsistentRingQueueBalancer(
IConsistentRingProviderForGrains ringProvider,
IStreamQueueMapper queueMapper)
{
if (ringProvider == null)
{
throw new ArgumentNullException("ringProvider");
}
if (queueMapper == null)
{
throw new ArgumentNullException("queueMapper");
}
if (!(queueMapper is IConsistentRingStreamQueueMapper))
{
throw new ArgumentException("queueMapper for ConsistentRingQueueBalancer should implement IConsistentRingStreamQueueMapper", "queueMapper");
}
streamQueueMapper = (IConsistentRingStreamQueueMapper)queueMapper;
myRange = ringProvider.GetMyRange();
ringProvider.SubscribeToRangeChangeEvents(this);
}
