/// <summary> Init the factory.</summary>
public virtual void Init()
{
this.streamQueueMapper = new HashRingBasedStreamQueueMapper(this.options.NumQueues, providerName);
this.adapterCache = new SimpleQueueAdapterCache(this.options.CacheSize, this.providerName, this.loggerFactory);
this.StreamFailureHandlerFactory = this.StreamFailureHandlerFactory ??
((qid) => Task.FromResult <IStreamFailureHandler>(new NoOpStreamDeliveryFailureHandler()));
}
/// <summary> Init the factory.</summary>
public virtual void Init(IProviderConfiguration config, string providerName, Logger logger, IServiceProvider serviceProvider)
{
if (config == null) throw new ArgumentNullException("config");
if (!config.Properties.TryGetValue(DataConnectionStringPropertyName, out dataConnectionString))
throw new ArgumentException(String.Format("{0} property not set", DataConnectionStringPropertyName));
if (!config.Properties.TryGetValue(DeploymentIdPropertyName, out deploymentId))
throw new ArgumentException(String.Format("{0} property not set", DeploymentIdPropertyName));
cacheSize = SimpleQueueAdapterCache.ParseSize(config, CacheSizeDefaultValue);
string numQueuesString;
numQueues = NumQueuesDefaultValue;
if (config.Properties.TryGetValue(NumQueuesPropertyName, out numQueuesString))
{
if (!int.TryParse(numQueuesString, out numQueues))
throw new ArgumentException(String.Format("{0} invalid. Must be int", NumQueuesPropertyName));
}
this.providerName = providerName;
streamQueueMapper = new HashRingBasedStreamQueueMapper(numQueues, providerName);
adapterCache = new SimpleQueueAdapterCache(cacheSize, logger);
if (StreamFailureHandlerFactory == null)
{
StreamFailureHandlerFactory =
qid => Task.FromResult<IStreamFailureHandler>(new NoOpStreamDeliveryFailureHandler(false));
}
}
/// <summary> Init the factory.</summary>
public virtual void Init(IProviderConfiguration config, string providerName, Logger logger)
{
if (config == null) throw new ArgumentNullException("config");
if (!config.Properties.TryGetValue(DATA_CONNECTION_STRING, out dataConnectionString))
throw new ArgumentException(String.Format("{0} property not set", DATA_CONNECTION_STRING));
if (!config.Properties.TryGetValue(DEPLOYMENT_ID, out deploymentId))
throw new ArgumentException(String.Format("{0} property not set", DEPLOYMENT_ID));
string cacheSizeString;
cacheSize = DEFAULT_CACHE_SIZE;
if (config.Properties.TryGetValue(CACHE_SIZE_PARAM, out cacheSizeString))
{
if (!int.TryParse(cacheSizeString, out cacheSize))
throw new ArgumentException(String.Format("{0} invalid. Must be int", CACHE_SIZE_PARAM));
}
string numQueuesString;
numQueues = DEFAULT_NUM_QUEUES;
if (config.Properties.TryGetValue(NUM_QUEUES_PARAM, out numQueuesString))
{
if (!int.TryParse(numQueuesString, out numQueues))
throw new ArgumentException(String.Format("{0} invalid. Must be int", NUM_QUEUES_PARAM));
}
this.providerName = providerName;
streamQueueMapper = new HashRingBasedStreamQueueMapper(numQueues, providerName);
adapterCache = new SimpleQueueAdapterCache(this, cacheSize, logger);
}
public void Init(IProviderConfiguration config, string providerName, Logger logger, IServiceProvider serviceProvider)
{
if (config == null)
{
throw new ArgumentNullException(nameof(config));
}
if (logger == null)
{
throw new ArgumentNullException(nameof(logger));
}
if (String.IsNullOrEmpty(providerName))
{
throw new ArgumentNullException(nameof(providerName));
}
// Creating an options object with all the config values
_options = new KafkaStreamProviderOptions(config);
if (!_options.UsingExternalMetrics)
{
Metric.Config.WithHttpEndpoint($"http://localhost:{_options.MetricsPort}/");
}
if (!_options.IncludeMetrics)
{
Metric.Context("KafkaStreamProvider").Advanced.CompletelyDisableMetrics();
}
_providerName = providerName;
_streamQueueMapper = new HashRingBasedStreamQueueMapper(_options.NumOfQueues, providerName);
_logger = logger;
_adapter = new KafkaQueueAdapter(_streamQueueMapper, _options, providerName, new KafkaBatchFactory(), _logger);
_adapterCache = new TimedQueueAdapterCache(this, TimeSpan.FromSeconds(_options.CacheTimespanInSeconds), _options.CacheSize, _options.CacheNumOfBuckets, logger);
}
/// <summary> Init the factory.</summary>
public virtual void Init(IProviderConfiguration config, string providerName, Logger logger, IServiceProvider serviceProvider)
{
if (config == null)
{
throw new ArgumentNullException("config");
}
if (!config.Properties.TryGetValue(DataConnectionStringPropertyName, out dataConnectionString))
{
throw new ArgumentException(string.Format("{0} property not set", DataConnectionStringPropertyName));
}
if (!config.Properties.TryGetValue(DeploymentIdPropertyName, out deploymentId))
{
throw new ArgumentException(string.Format("{0} property not set", DeploymentIdPropertyName));
}
cacheSize = SimpleQueueAdapterCache.ParseSize(config, CacheSizeDefaultValue);
numQueues = config.GetIntProperty(NumQueuesPropertyName, NumQueuesDefaultValue);
this.providerName = providerName;
streamQueueMapper = new HashRingBasedStreamQueueMapper(numQueues, providerName);
adapterCache = new SimpleQueueAdapterCache(cacheSize, logger);
if (StreamFailureHandlerFactory == null)
{
StreamFailureHandlerFactory =
qid => Task.FromResult <IStreamFailureHandler>(new NoOpStreamDeliveryFailureHandler(false));
}
}
/// <summary> Init the factory.</summary>
public virtual void Init(IProviderConfiguration config, string providerName, Logger logger, IServiceProvider serviceProvider)
{
if (config == null)
{
throw new ArgumentNullException("config");
}
if (!config.Properties.TryGetValue(DATA_CONNECTION_STRING, out dataConnectionString))
{
throw new ArgumentException(String.Format("{0} property not set", DATA_CONNECTION_STRING));
}
if (!config.Properties.TryGetValue(DEPLOYMENT_ID, out deploymentId))
{
throw new ArgumentException(String.Format("{0} property not set", DEPLOYMENT_ID));
}
cacheSize = SimpleQueueAdapterCache.ParseSize(config, DEFAULT_CACHE_SIZE);
string numQueuesString;
numQueues = DEFAULT_NUM_QUEUES;
if (config.Properties.TryGetValue(NUM_QUEUES_PARAM, out numQueuesString))
{
if (!int.TryParse(numQueuesString, out numQueues))
{
throw new ArgumentException(String.Format("{0} invalid. Must be int", NUM_QUEUES_PARAM));
}
}
this.providerName = providerName;
streamQueueMapper = new HashRingBasedStreamQueueMapper(numQueues, providerName);
adapterCache = new SimpleQueueAdapterCache(cacheSize, logger);
}
/// <summary> Init the factory.</summary>
public virtual void Init(IProviderConfiguration config, string providerName, Logger logger, IServiceProvider serviceProvider)
{
if (config == null)
{
throw new ArgumentNullException("config");
}
if (!config.Properties.TryGetValue(DataConnectionStringPropertyName, out dataConnectionString))
{
throw new ArgumentException(String.Format("{0} property not set", DataConnectionStringPropertyName));
}
if (!config.Properties.TryGetValue(DeploymentIdPropertyName, out deploymentId))
{
throw new ArgumentException(String.Format("{0} property not set", DeploymentIdPropertyName));
}
cacheSize = SimpleQueueAdapterCache.ParseSize(config, CacheSizeDefaultValue);
string numQueuesString;
numQueues = NumQueuesDefaultValue;
if (config.Properties.TryGetValue(NumQueuesPropertyName, out numQueuesString))
{
if (!int.TryParse(numQueuesString, out numQueues))
{
throw new ArgumentException(String.Format("{0} invalid. Must be int", NumQueuesPropertyName));
}
}
this.providerName = providerName;
streamQueueMapper = new HashRingBasedStreamQueueMapper(numQueues, providerName);
adapterCache = new SimpleQueueAdapterCache(cacheSize, logger);
}
public void Init(IProviderConfiguration config, string providerName, Logger logger, IServiceProvider serviceProvider)
{
if (config == null) throw new ArgumentNullException(nameof(config));
if (logger == null) throw new ArgumentNullException(nameof(logger));
if (String.IsNullOrEmpty(providerName)) throw new ArgumentNullException(nameof(providerName));
// Creating an options object with all the config values
_options = new KafkaStreamProviderOptions(config);
if (!_options.UsingExternalMetrics)
{
Metric.Config.WithHttpEndpoint($"http://localhost:{_options.MetricsPort}/");
}
if (!_options.IncludeMetrics)
{
Metric.Context("KafkaStreamProvider").Advanced.CompletelyDisableMetrics();
}
_providerName = providerName;
_streamQueueMapper = new HashRingBasedStreamQueueMapper(_options.NumOfQueues, providerName);
_logger = logger;
_adapter = new KafkaQueueAdapter(_streamQueueMapper, _options, providerName, new KafkaBatchFactory(), _logger);
_adapterCache = new TimedQueueAdapterCache(this, TimeSpan.FromSeconds(_options.CacheTimespanInSeconds), _options.CacheSize, _options.CacheNumOfBuckets, logger);
}
public KafkaAdapterFactory(
string name,
KafkaStreamOptions options,
SimpleQueueCacheOptions cacheOptions,
SerializationManager serializationManager,
ILoggerFactory loggerFactory,
IGrainFactory grainFactory
)
{
_options = options ?? throw new ArgumentNullException(nameof(options));
_name = name;
_serializationManager = serializationManager;
_loggerFactory = loggerFactory;
_grainFactory = grainFactory;
_logger = loggerFactory.CreateLogger <KafkaAdapterFactory>();
if (options.Topics != null && options.Topics.Count == 0)
{
throw new ArgumentNullException(nameof(options.Topics));
}
_adapterCache = new SimpleQueueAdapterCache(
cacheOptions,
name,
loggerFactory
);
_queueProperties = GetQueuesProperties();
_streamQueueMapper = new ExternalQueueMapper(_queueProperties.Values);
}
public void Init(IProviderConfiguration config, string providerName, Logger logger, IServiceProvider serviceProvider)
{
if (config == null)
{
throw new ArgumentNullException(nameof(config));
}
if (providerName == null)
{
throw new ArgumentNullException(nameof(providerName));
}
if (logger == null)
{
throw new ArgumentNullException(nameof(logger));
}
if (serviceProvider == null)
{
throw new ArgumentNullException(nameof(serviceProvider));
}
m_ProviderName = providerName;
m_Logger = logger;
m_QueueMapper = new EventStoreQueueMapper(1, providerName);
m_QueueAdapterCache = new EventStoreQueueAdapterCache(this, m_Logger);
m_ConnectionString = config.Properties["ConnectionString"];
CreateConnection();
}
/// <summary> Init the factory.</summary>
public virtual void Init(IProviderConfiguration config, string providerName, Logger logger, IServiceProvider serviceProvider)
{
if (config == null)
{
throw new ArgumentNullException(nameof(config));
}
if (!config.Properties.TryGetValue(AzureQueueAdapterConstants.DataConnectionStringPropertyName, out dataConnectionString))
{
throw new ArgumentException($"{AzureQueueAdapterConstants.DataConnectionStringPropertyName} property not set");
}
if (!config.Properties.TryGetValue(AzureQueueAdapterConstants.DeploymentIdPropertyName, out deploymentId))
{
throw new ArgumentException($"{AzureQueueAdapterConstants.DeploymentIdPropertyName} property not set");
}
string messageVisibilityTimeoutRaw;
if (config.Properties.TryGetValue(AzureQueueAdapterConstants.MessageVisibilityTimeoutPropertyName, out messageVisibilityTimeoutRaw))
{
TimeSpan messageVisibilityTimeoutTemp;
if (!TimeSpan.TryParse(messageVisibilityTimeoutRaw, out messageVisibilityTimeoutTemp))
{
throw new ArgumentException(
$"Failed to parse {AzureQueueAdapterConstants.MessageVisibilityTimeoutPropertyName} value '{messageVisibilityTimeoutRaw}' as a TimeSpan");
}
messageVisibilityTimeout = messageVisibilityTimeoutTemp;
}
else
{
messageVisibilityTimeout = null;
}
cacheSize = SimpleQueueAdapterCache.ParseSize(config, AzureQueueAdapterConstants.CacheSizeDefaultValue);
this.loggerFactory = serviceProvider.GetRequiredService <ILoggerFactory>();
string numQueuesString;
numQueues = AzureQueueAdapterConstants.NumQueuesDefaultValue;
if (config.Properties.TryGetValue(AzureQueueAdapterConstants.NumQueuesPropertyName, out numQueuesString))
{
if (!int.TryParse(numQueuesString, out numQueues))
{
throw new ArgumentException($"{AzureQueueAdapterConstants.NumQueuesPropertyName} invalid. Must be int");
}
}
this.providerName = providerName;
streamQueueMapper = new HashRingBasedStreamQueueMapper(numQueues, providerName);
adapterCache = new SimpleQueueAdapterCache(cacheSize, logger);
if (StreamFailureHandlerFactory == null)
{
StreamFailureHandlerFactory =
qid => Task.FromResult <IStreamFailureHandler>(new NoOpStreamDeliveryFailureHandler());
}
this.SerializationManager = serviceProvider.GetRequiredService <SerializationManager>();
this.adaptorFactory = () => ActivatorUtilities.GetServiceOrCreateInstance <TDataAdapter>(serviceProvider);
}
public virtual void Init()
{
this._streamQueueMapper = new HashRingBasedStreamQueueMapper(this.options.NumSubscriptions, this._providerName);
this._adapterCache = new SimpleQueueAdapterCache(this.options.CacheSize, this._providerName, loggerFactory);
if (StreamFailureHandlerFactory == null)
{
StreamFailureHandlerFactory =
qid => Task.FromResult <IStreamFailureHandler>(new NoOpStreamDeliveryFailureHandler());
}
}
private void InitializeInternal(IQueueAdapter qAdapter, IQueueAdapterCache queueAdapterCache, IStreamFailureHandler failureHandler)
{
logger.Info(ErrorCode.PersistentStreamPullingAgent_02, "Init of {0} {1} on silo {2} for queue {3}.",
GetType().Name, ((ISystemTargetBase)this).GrainId.ToDetailedString(), Silo, QueueId.ToStringWithHashCode());
// Remove cast once we cleanup
queueAdapter = qAdapter;
streamFailureHandler = failureHandler;
lastTimeCleanedPubSubCache = DateTime.UtcNow;
try
{
receiver = queueAdapter.CreateReceiver(QueueId);
}
catch (Exception exc)
{
logger.Error(ErrorCode.PersistentStreamPullingAgent_02, "Exception while calling IQueueAdapter.CreateNewReceiver.", exc);
throw;
}
try
{
if (queueAdapterCache != null)
{
queueCache = queueAdapterCache.CreateQueueCache(QueueId);
}
}
catch (Exception exc)
{
logger.Error(ErrorCode.PersistentStreamPullingAgent_23, "Exception while calling IQueueAdapterCache.CreateQueueCache.", exc);
throw;
}
try
{
receiverInitTask = OrleansTaskExtentions.SafeExecute(() => receiver.Initialize(config.InitQueueTimeout))
.LogException(logger, ErrorCode.PersistentStreamPullingAgent_03, $"QueueAdapterReceiver {QueueId.ToStringWithHashCode()} failed to Initialize.");
receiverInitTask.Ignore();
}
catch
{
// Just ignore this exception and proceed as if Initialize has succeeded.
// We already logged individual exceptions for individual calls to Initialize. No need to log again.
}
// Setup a reader for a new receiver.
// Even if the receiver failed to initialise, treat it as OK and start pumping it. It's receiver responsibility to retry initialization.
var randomTimerOffset = safeRandom.NextTimeSpan(config.GetQueueMsgsTimerPeriod);
timer = RegisterTimer(AsyncTimerCallback, QueueId, randomTimerOffset, config.GetQueueMsgsTimerPeriod);
IntValueStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_PUBSUB_CACHE_SIZE, StatisticUniquePostfix), () => pubSubCache.Count);
logger.Info((int)ErrorCode.PersistentStreamPullingAgent_04, "Taking queue {0} under my responsibility.", QueueId.ToStringWithHashCode());
}
/// <summary> Init the factory.</summary>
public virtual void Init(IProviderConfiguration config, string providerName, Logger logger, IServiceProvider serviceProvider)
{
if (config == null)
{
throw new ArgumentNullException("config");
}
if (!config.Properties.TryGetValue(DataConnectionStringPropertyName, out dataConnectionString))
{
throw new ArgumentException(String.Format("{0} property not set", DataConnectionStringPropertyName));
}
if (!config.Properties.TryGetValue(DeploymentIdPropertyName, out deploymentId))
{
throw new ArgumentException(String.Format("{0} property not set", DeploymentIdPropertyName));
}
string messageVisibilityTimeoutRaw;
if (config.Properties.TryGetValue(MessageVisibilityTimeoutPropertyName, out messageVisibilityTimeoutRaw))
{
TimeSpan messageVisibilityTimeoutTemp;
if (!TimeSpan.TryParse(messageVisibilityTimeoutRaw, out messageVisibilityTimeoutTemp))
{
throw new ArgumentException(String.Format("Failed to parse {0} value '{1}' as a TimeSpan",
MessageVisibilityTimeoutPropertyName, messageVisibilityTimeoutRaw));
}
messageVisibilityTimeout = messageVisibilityTimeoutTemp;
}
else
{
messageVisibilityTimeout = null;
}
cacheSize = SimpleQueueAdapterCache.ParseSize(config, CacheSizeDefaultValue);
string numQueuesString;
numQueues = NumQueuesDefaultValue;
if (config.Properties.TryGetValue(NumQueuesPropertyName, out numQueuesString))
{
if (!int.TryParse(numQueuesString, out numQueues))
{
throw new ArgumentException(String.Format("{0} invalid. Must be int", NumQueuesPropertyName));
}
}
this.providerName = providerName;
streamQueueMapper = new HashRingBasedStreamQueueMapper(numQueues, providerName);
adapterCache = new SimpleQueueAdapterCache(cacheSize, logger);
if (StreamFailureHandlerFactory == null)
{
StreamFailureHandlerFactory =
qid => Task.FromResult <IStreamFailureHandler>(new NoOpStreamDeliveryFailureHandler(false));
}
}
public SimpleQueueAdapterCacheCursor(IQueueAdapterCache cache, Guid streamGuid, string streamNamespace, EventSequenceToken sequenceToken)
{
if (cache == null)
{
throw new ArgumentNullException("cache");
}
this.cache = cache;
this.streamGuid = streamGuid;
this.streamNamespace = streamNamespace;
cursor = this.cache.GetCursor(sequenceToken);
}
public void Init(IProviderConfiguration config, string providerName, IServiceProvider serviceProvider)
{
Name = providerName;
_publishSubscribe = serviceProvider.GetRequiredService <IPublishSubscribe>();
_serializationManager = serviceProvider.GetRequiredService <SerializationManager>();
ILoggerFactory loggerFactory = serviceProvider.GetRequiredService <ILoggerFactory>();
PublishSubscribeStreamConfiguration publishSubscribeStreamConfiguration = new PublishSubscribeStreamConfiguration(config);
_adapterCache = new SimpleQueueAdapterCache(publishSubscribeStreamConfiguration.InMemoryCacheSize, providerName, loggerFactory);
_streamQueueMapper = new HashRingBasedStreamQueueMapper(publishSubscribeStreamConfiguration.NumberOfQueues, providerName);
}
private AzureQueueAdapterReceiver(QueueId queueId, AzureQueueDataManager queue, int cacheSize)
{
if (queueId == null)
{
throw new ArgumentNullException("queueId");
}
if (queue == null)
{
throw new ArgumentNullException("queue");
}
Id = queueId;
this.queue = queue;
cache = new SimpleQueueAdapterCache(cacheSize);
}
internal PersistentStreamPullingManager(
SystemTargetGrainId managerId,
string strProviderName,
IStreamProviderRuntime runtime,
IStreamPubSub streamPubSub,
IQueueAdapterFactory adapterFactory,
IStreamQueueBalancer streamQueueBalancer,
IStreamFilter streamFilter,
StreamPullingAgentOptions options,
ILoggerFactory loggerFactory,
SiloAddress siloAddress)
: base(managerId, siloAddress, lowPriority: false, loggerFactory)
{
if (string.IsNullOrWhiteSpace(strProviderName))
{
throw new ArgumentNullException("strProviderName");
}
if (runtime == null)
{
throw new ArgumentNullException("runtime", "IStreamProviderRuntime runtime reference should not be null");
}
if (streamPubSub == null)
{
throw new ArgumentNullException("streamPubSub", "StreamPubSub reference should not be null");
}
if (streamQueueBalancer == null)
{
throw new ArgumentNullException("streamQueueBalancer", "IStreamQueueBalancer streamQueueBalancer reference should not be null");
}
queuesToAgentsMap = new Dictionary <QueueId, PersistentStreamPullingAgent>();
streamProviderName = strProviderName;
providerRuntime = runtime;
pubSub = streamPubSub;
this.options = options;
nonReentrancyGuarantor = new AsyncSerialExecutor();
latestRingNotificationSequenceNumber = 0;
latestCommandNumber = 0;
queueBalancer = streamQueueBalancer;
this.streamFilter = streamFilter;
this.adapterFactory = adapterFactory;
queueAdapterCache = adapterFactory.GetQueueAdapterCache();
logger = loggerFactory.CreateLogger($"{GetType().FullName}.{streamProviderName}");
Log(ErrorCode.PersistentStreamPullingManager_01, "Created {0} for Stream Provider {1}.", GetType().Name, streamProviderName);
this.loggerFactory = loggerFactory;
IntValueStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_PULLING_AGENTS, strProviderName), () => queuesToAgentsMap.Count);
}
internal PersistentStreamPullingManager(
GrainId id,
string strProviderName,
IStreamProviderRuntime runtime,
IStreamPubSub streamPubSub,
IQueueAdapterFactory adapterFactory,
IStreamQueueBalancer streamQueueBalancer,
TimeSpan queueGetPeriod,
TimeSpan initQueueTimeout,
TimeSpan maxEvenDeliveryTime)
: base(id, runtime.ExecutingSiloAddress)
{
if (string.IsNullOrWhiteSpace(strProviderName))
{
throw new ArgumentNullException("strProviderName");
}
if (runtime == null)
{
throw new ArgumentNullException("runtime", "IStreamProviderRuntime runtime reference should not be null");
}
if (streamPubSub == null)
{
throw new ArgumentNullException("streamPubSub", "StreamPubSub reference should not be null");
}
if (streamQueueBalancer == null)
{
throw new ArgumentNullException("streamQueueBalancer", "IStreamQueueBalancer streamQueueBalancer reference should not be null");
}
queuesToAgentsMap = new Dictionary <QueueId, PersistentStreamPullingAgent>();
streamProviderName = strProviderName;
providerRuntime = runtime;
pubSub = streamPubSub;
this.queueGetPeriod = queueGetPeriod;
this.initQueueTimeout = initQueueTimeout;
this.maxEvenDeliveryTime = maxEvenDeliveryTime;
nonReentrancyGuarantor = new AsyncSerialExecutor();
latestRingNotificationSequenceNumber = 0;
latestCommandNumber = 0;
queueBalancer = streamQueueBalancer;
this.adapterFactory = adapterFactory;
queueAdapterCache = adapterFactory.GetQueueAdapterCache();
logger = providerRuntime.GetLogger(GetType().Name + "-" + streamProviderName);
logger.Info((int)ErrorCode.PersistentStreamPullingManager_01, "Created {0} for Stream Provider {1}.", GetType().Name, streamProviderName);
IntValueStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_PULLING_AGENTS, strProviderName), () => queuesToAgentsMap.Count);
}
public AzureQueueAdapterFactory(
string name,
AzureQueueOptions options,
SimpleQueueCacheOptions cacheOptions,
IQueueDataAdapter <string, IBatchContainer> dataAdapter,
IOptions <ClusterOptions> clusterOptions,
ILoggerFactory loggerFactory)
{
this.providerName = name;
this.options = options ?? throw new ArgumentNullException(nameof(options));
this.dataAdapter = dataAdapter ?? throw new ArgumentNullException(nameof(dataAdapter));;
this.clusterOptions = clusterOptions.Value;
this.loggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory));
this.streamQueueMapper = new AzureStreamQueueMapper(options.QueueNames, providerName);
this.adapterCache = new SimpleQueueAdapterCache(cacheOptions, this.providerName, this.loggerFactory);
}
public ServiceBusAdapterFactory(
ILoggerFactory loggerFactory,
string name,
ServiceBusOptions options,
SimpleQueueCacheOptions cacheOptions,
IOptions <ClusterOptions> clusterOptions,
SerializationManager serializationManager)
{
_providerName = name;
_options = options ?? throw new ArgumentNullException(nameof(options));
_clusterOptions = clusterOptions.Value;
SerializationManager = serializationManager ?? throw new ArgumentNullException(nameof(serializationManager));
_loggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory));
_streamQueueMapper = new ServiceBusStreamQueueMapper(options, $"{options.QueuePrefix}-{name}");
_adapterCache = new SimpleQueueAdapterCache(cacheOptions, _providerName, _loggerFactory);
}
public SQSAdapterFactory(
string name,
SqsOptions sqsOptions,
HashRingStreamQueueMapperOptions queueMapperOptions,
SimpleQueueCacheOptions cacheOptions,
IOptions <ClusterOptions> clusterOptions,
Orleans.Serialization.Serializer serializer,
ILoggerFactory loggerFactory)
{
this.providerName = name;
this.sqsOptions = sqsOptions;
this.clusterOptions = clusterOptions.Value;
this.serializer = serializer.GetSerializer <SQSBatchContainer>();
this.loggerFactory = loggerFactory;
streamQueueMapper = new HashRingBasedStreamQueueMapper(queueMapperOptions, this.providerName);
adapterCache = new SimpleQueueAdapterCache(cacheOptions, this.providerName, this.loggerFactory);
}
public PubSubAdapterFactory(
string name,
PubSubOptions options,
HashRingStreamQueueMapperOptions queueMapperOptions,
SimpleQueueCacheOptions cacheOptions,
IServiceProvider serviceProvider,
IOptions <ClusterOptions> clusterOptions,
ILoggerFactory loggerFactory)
{
this._providerName = name;
this.options = options;
this.clusterOptions = clusterOptions.Value;
this.loggerFactory = loggerFactory;
this._adaptorFactory = () => ActivatorUtilities.GetServiceOrCreateInstance <TDataAdapter>(serviceProvider);
this._streamQueueMapper = new HashRingBasedStreamQueueMapper(queueMapperOptions, this._providerName);
this._adapterCache = new SimpleQueueAdapterCache(cacheOptions, this._providerName, loggerFactory);
}
internal PersistentStreamPullingManager(
GrainId id,
string strProviderName,
IStreamProviderRuntime runtime,
IStreamPubSub streamPubSub,
IQueueAdapterFactory adapterFactory,
IStreamQueueBalancer streamQueueBalancer,
PersistentStreamProviderConfig config)
: base(id, runtime.ExecutingSiloAddress)
{
if (string.IsNullOrWhiteSpace(strProviderName))
{
throw new ArgumentNullException("strProviderName");
}
if (runtime == null)
{
throw new ArgumentNullException("runtime", "IStreamProviderRuntime runtime reference should not be null");
}
if (streamPubSub == null)
{
throw new ArgumentNullException("streamPubSub", "StreamPubSub reference should not be null");
}
if (streamQueueBalancer == null)
{
throw new ArgumentNullException("streamQueueBalancer", "IStreamQueueBalancer streamQueueBalancer reference should not be null");
}
queuesToAgentsMap = new Dictionary<QueueId, PersistentStreamPullingAgent>();
streamProviderName = strProviderName;
providerRuntime = runtime;
pubSub = streamPubSub;
this.config = config;
nonReentrancyGuarantor = new AsyncSerialExecutor();
latestRingNotificationSequenceNumber = 0;
latestCommandNumber = 0;
queueBalancer = streamQueueBalancer;
this.adapterFactory = adapterFactory;
queueAdapterCache = adapterFactory.GetQueueAdapterCache();
logger = TraceLogger.GetLogger(GetType().Name + "-" + streamProviderName, TraceLogger.LoggerType.Provider);
Log(ErrorCode.PersistentStreamPullingManager_01, "Created {0} for Stream Provider {1}.", GetType().Name, streamProviderName);
IntValueStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_PULLING_AGENTS, strProviderName), () => queuesToAgentsMap.Count);
queuePrintTimer = base.RegisterTimer(AsyncTimerCallback, null, QUEUES_PRINT_PERIOD, QUEUES_PRINT_PERIOD);
}
internal PersistentStreamPullingManager(
GrainId id,
string strProviderName,
IStreamProviderRuntime runtime,
IStreamPubSub streamPubSub,
IQueueAdapterFactory adapterFactory,
IStreamQueueBalancer streamQueueBalancer,
PersistentStreamProviderConfig config)
: base(id, runtime.ExecutingSiloAddress)
{
if (string.IsNullOrWhiteSpace(strProviderName))
{
throw new ArgumentNullException("strProviderName");
}
if (runtime == null)
{
throw new ArgumentNullException("runtime", "IStreamProviderRuntime runtime reference should not be null");
}
if (streamPubSub == null)
{
throw new ArgumentNullException("streamPubSub", "StreamPubSub reference should not be null");
}
if (streamQueueBalancer == null)
{
throw new ArgumentNullException("streamQueueBalancer", "IStreamQueueBalancer streamQueueBalancer reference should not be null");
}
queuesToAgentsMap = new Dictionary <QueueId, PersistentStreamPullingAgent>();
streamProviderName = strProviderName;
providerRuntime = runtime;
pubSub = streamPubSub;
this.config = config;
nonReentrancyGuarantor = new AsyncSerialExecutor();
latestRingNotificationSequenceNumber = 0;
latestCommandNumber = 0;
queueBalancer = streamQueueBalancer;
this.adapterFactory = adapterFactory;
queueAdapterCache = adapterFactory.GetQueueAdapterCache();
logger = LogManager.GetLogger(GetType().Name + "-" + streamProviderName, LoggerType.Provider);
Log(ErrorCode.PersistentStreamPullingManager_01, "Created {0} for Stream Provider {1}.", GetType().Name, streamProviderName);
IntValueStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_PULLING_AGENTS, strProviderName), () => queuesToAgentsMap.Count);
queuePrintTimer = base.RegisterTimer(AsyncTimerCallback, null, QUEUES_PRINT_PERIOD, QUEUES_PRINT_PERIOD);
}
public RabbitMqAdapterFactory(
string providerName,
IOptionsMonitor <RabbitMqOptions> rmqOptionsAccessor,
IOptionsMonitor <SimpleQueueCacheOptions> cachingOptionsAccessor,
IServiceProvider serviceProvider,
ILoggerFactory loggerFactory,
IRabbitMqStreamQueueMapperFactory streamQueueMapperFactory)
{
if (string.IsNullOrEmpty(providerName))
{
throw new ArgumentNullException(nameof(providerName));
}
if (rmqOptionsAccessor == null)
{
throw new ArgumentNullException(nameof(rmqOptionsAccessor));
}
if (cachingOptionsAccessor == null)
{
throw new ArgumentNullException(nameof(cachingOptionsAccessor));
}
if (serviceProvider == null)
{
throw new ArgumentNullException(nameof(serviceProvider));
}
if (loggerFactory == null)
{
throw new ArgumentNullException(nameof(loggerFactory));
}
if (streamQueueMapperFactory == null)
{
throw new ArgumentNullException(nameof(streamQueueMapperFactory));
}
var rmqOptions = rmqOptionsAccessor.Get(providerName);
var cachingOptions = cachingOptionsAccessor.Get(providerName);
_cache = new SimpleQueueAdapterCache(cachingOptions, providerName, loggerFactory);
_mapper = streamQueueMapperFactory.Get(providerName);
_failureHandler = Task.FromResult <IStreamFailureHandler>(new NoOpStreamDeliveryFailureHandler(false));
var dataAdapter = serviceProvider.GetServiceByName <IQueueDataAdapter <RabbitMqMessage, IEnumerable <IBatchContainer> > >(providerName) ??
RabbitMqDataAdapter.Create(serviceProvider, providerName);
_adapter = new RabbitMqAdapter(rmqOptions, dataAdapter, providerName, _mapper, serviceProvider.GetRequiredServiceByName <IRabbitMqConnectorFactory>(providerName));
}
public AzureQueueAdapterFactory(
string name,
AzureQueueOptions options,
SimpleQueueCacheOptions cacheOptions,
IServiceProvider serviceProvider,
IOptions <ClusterOptions> clusterOptions,
SerializationManager serializationManager,
ILoggerFactory loggerFactory)
{
this.providerName = name;
this.options = options ?? throw new ArgumentNullException(nameof(options));
this.clusterOptions = clusterOptions.Value;
this.SerializationManager = serializationManager ?? throw new ArgumentNullException(nameof(serializationManager));
this.loggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory));
this.dataAadaptorFactory = () => ActivatorUtilities.GetServiceOrCreateInstance <TDataAdapter>(serviceProvider);
this.streamQueueMapper = new AzureStreamQueueMapper(options.QueueNames, providerName);
this.adapterCache = new SimpleQueueAdapterCache(cacheOptions, this.providerName, this.loggerFactory);
}
public TestAdapterFactory(
string name,
SimpleQueueCacheOptions cacheOptions,
Func <IEnumerable <IBatchContainer> > queueMessagesProvider,
Action <IEnumerable <IBatchContainer> > onMessagesDelivered,
ILoggerFactory loggerFactory
)
{
_providerName = name;
_queueMessagesProvider = queueMessagesProvider ?? throw new ArgumentNullException(nameof(queueMessagesProvider));
_onMessagesDelivered = onMessagesDelivered ?? throw new ArgumentNullException(nameof(onMessagesDelivered));
_loggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory));
_streamQueueMapper = new HashRingBasedStreamQueueMapper(new HashRingStreamQueueMapperOptions()
{
TotalQueueCount = 1
}, _providerName);
_adapterCache = new SimpleQueueAdapterCache(cacheOptions, _providerName, _loggerFactory);
}
public SQSAdapterFactory(
string name,
SqsOptions sqsOptions,
HashRingStreamQueueMapperOptions queueMapperOptions,
SimpleQueueCacheOptions cacheOptions,
IServiceProvider serviceProvider,
IOptions <ClusterOptions> clusterOptions,
SerializationManager serializationManager,
ILoggerFactory loggerFactory)
{
this.providerName = name;
this.sqsOptions = sqsOptions;
this.clusterOptions = clusterOptions.Value;
this.serializationManager = serializationManager;
this.loggerFactory = loggerFactory;
streamQueueMapper = new HashRingBasedStreamQueueMapper(queueMapperOptions, this.providerName);
adapterCache = new SimpleQueueAdapterCache(cacheOptions, this.providerName, this.loggerFactory);
}
public RabbitMQAdapterFactory(
string name,
RabbitMQStreamProviderOptions options,
HashRingStreamQueueMapperOptions queueMapperOptions,
SimpleQueueCacheOptions cacheOptions,
IServiceProvider serviceProvider,
IOptions <ClusterOptions> clusterOptions,
SerializationManager serializationManager,
ILoggerFactory loggerFactory)
{
providerName = name;
this.options = options ?? throw new ArgumentNullException(nameof(options));
this.clusterOptions = clusterOptions.Value;
//this.SerializationManager = serializationManager ?? throw new ArgumentNullException(nameof(serializationManager));
this.loggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory));
streamQueueMapper = new HashRingBasedStreamQueueMapper(queueMapperOptions, providerName);
adapterCache = new SimpleQueueAdapterCache(cacheOptions, providerName, this.loggerFactory);
mapper = ActivatorUtilities.GetServiceOrCreateInstance <TMapper>(serviceProvider);
}
internal PersistentStreamPullingAgent(
SystemTargetGrainId id,
string strProviderName,
ILoggerFactory loggerFactory,
IStreamPubSub streamPubSub,
IStreamFilter streamFilter,
QueueId queueId,
StreamPullingAgentOptions options,
SiloAddress siloAddress,
IQueueAdapter queueAdapter,
IQueueAdapterCache queueAdapterCache,
IStreamFailureHandler streamFailureHandler)
: base(id, siloAddress, true, loggerFactory)
{
if (strProviderName == null)
{
throw new ArgumentNullException("runtime", "PersistentStreamPullingAgent: strProviderName should not be null");
}
QueueId = queueId;
streamProviderName = strProviderName;
pubSub = streamPubSub;
this.streamFilter = streamFilter;
pubSubCache = new Dictionary <InternalStreamId, StreamConsumerCollection>();
this.options = options;
this.queueAdapter = queueAdapter ?? throw new ArgumentNullException(nameof(queueAdapter));
this.streamFailureHandler = streamFailureHandler ?? throw new ArgumentNullException(nameof(streamFailureHandler));;
this.queueAdapterCache = queueAdapterCache;
numMessages = 0;
logger = loggerFactory.CreateLogger($"{this.GetType().Namespace}.{streamProviderName}");
logger.LogInformation(
(int)ErrorCode.PersistentStreamPullingAgent_01,
"Created {Name} {Id} for Stream Provider {StreamProvider} on silo {Silo} for Queue {Queue}.",
GetType().Name,
((ISystemTargetBase)this).GrainId.ToString(),
streamProviderName,
Silo,
QueueId.ToStringWithHashCode());
numReadMessagesCounter = CounterStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_READ_MESSAGES, StatisticUniquePostfix));
numSentMessagesCounter = CounterStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_SENT_MESSAGES, StatisticUniquePostfix));
}
public RabbitMqAdapterFactory(
string providerName,
RabbitMqOptions rmqOptions,
CachingOptions cachingOptions,
IServiceProvider serviceProvider,
ILoggerFactory loggerFactory)
{
if (string.IsNullOrEmpty(providerName))
{
throw new ArgumentNullException(nameof(providerName));
}
if (rmqOptions == null)
{
throw new ArgumentNullException(nameof(rmqOptions));
}
if (cachingOptions == null)
{
throw new ArgumentNullException(nameof(cachingOptions));
}
if (serviceProvider == null)
{
throw new ArgumentNullException(nameof(serviceProvider));
}
if (loggerFactory == null)
{
throw new ArgumentNullException(nameof(loggerFactory));
}
_cache = new SimpleQueueAdapterCache(new SimpleQueueCacheOptions {
CacheSize = cachingOptions.CacheSize
}, providerName, loggerFactory);
_mapper = new HashRingBasedStreamQueueMapper(new HashRingStreamQueueMapperOptions {
TotalQueueCount = rmqOptions.NumberOfQueues
}, rmqOptions.QueueNamePrefix);
_failureHandler = Task.FromResult <IStreamFailureHandler>(new NoOpStreamDeliveryFailureHandler(false));
var serializer = typeof(TSerializer) == typeof(DefaultBatchContainerSerializer)
? new DefaultBatchContainerSerializer(serviceProvider.GetRequiredService <SerializationManager>())
: (IBatchContainerSerializer) new TSerializer();
_adapter = new RabbitMqAdapter(rmqOptions, cachingOptions, serializer, _mapper, providerName, loggerFactory);
}
public void Init(IProviderConfiguration config, string providerName, Logger logger, IServiceProvider serviceProvider)
{
if (config == null)
{
throw new ArgumentNullException(nameof(config));
}
if (logger == null)
{
throw new ArgumentNullException(nameof(logger));
}
if (string.IsNullOrEmpty(providerName))
{
throw new ArgumentNullException(nameof(providerName));
}
_options = new RabbitMqStreamProviderOptions(config);
_providerName = providerName;
_cache = new ConcurrentQueueAdapterCache(_options.CacheSize);
_mapper = new HashRingBasedStreamQueueMapper(_options.NumberOfQueues, _options.QueueNamePrefix);
_failureHandler = Task.FromResult <IStreamFailureHandler>(new NoOpStreamDeliveryFailureHandler(false));
_adapter = new RabbitMqAdapter(_options, serviceProvider.GetRequiredService <SerializationManager>(), _mapper, _providerName, logger);
}
internal PersistentStreamPullingManager(
GrainId id,
string strProviderName,
IStreamProviderRuntime runtime,
IQueueAdapterFactory adapterFactory,
IStreamQueueBalancer streamQueueBalancer,
TimeSpan queueGetPeriod,
TimeSpan initQueueTimeout)
: base(id, runtime.ExecutingSiloAddress)
{
if (string.IsNullOrWhiteSpace(strProviderName))
{
throw new ArgumentNullException("strProviderName");
}
if (runtime == null)
{
throw new ArgumentNullException("runtime", "IStreamProviderRuntime runtime reference should not be null");
}
if (streamQueueBalancer == null)
{
throw new ArgumentNullException("streamQueueBalancer", "IStreamQueueBalancer streamQueueBalancer reference should not be null");
}
queuesToAgentsMap = new Dictionary<QueueId, PersistentStreamPullingAgent>();
streamProviderName = strProviderName;
providerRuntime = runtime;
this.queueGetPeriod = queueGetPeriod;
this.initQueueTimeout = initQueueTimeout;
nonReentrancyGuarantor = new AsyncSerialExecutor();
latestRingNotificationSequenceNumber = 0;
queueBalancer = streamQueueBalancer;
queueAdapterCache = adapterFactory.GetQueueAdapterCache();
logger = providerRuntime.GetLogger(GetType().Name + "-" + streamProviderName);
logger.Info((int)ErrorCode.PersistentStreamPullingManager_01, "Created {0} for Stream Provider {1}.", GetType().Name, streamProviderName);
IntValueStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_PULLING_AGENTS, strProviderName), () => queuesToAgentsMap.Count);
}
public KafkaQueueAdapterFactory(
string name,
KafkaStreamProviderOptions options,
ILoggerFactory loggerFactory)
{
if (string.IsNullOrEmpty(name))
{
throw new ArgumentNullException(nameof(name));
}
_options = options ?? throw new ArgumentNullException(nameof(options));
_loggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory));
_logger = _loggerFactory.CreateLogger <KafkaQueueAdapterFactory>();
_providerName = name;
_streamQueueMapper = new HashRingBasedStreamQueueMapper(
new HashRingStreamQueueMapperOptions {
TotalQueueCount = _options.PartitionsCount
},
name);
_adapterCache = new SimpleQueueAdapterCache(new SimpleQueueCacheOptions {
CacheSize = options.Cache.CacheSize
}, _providerName, _loggerFactory);
}
public IQueueAdapterCache GetQueueAdapterCache()
{
return _adapterCache ?? (_adapterCache = new Cache.QueueAdapterCache(this, TimeSpan.FromSeconds(_timeLimit), _logger));
}
private void InitializeInternal(IQueueAdapter qAdapter, IQueueAdapterCache queueAdapterCache, IStreamFailureHandler failureHandler)
{
logger.Info(ErrorCode.PersistentStreamPullingAgent_02, "Init of {0} {1} on silo {2} for queue {3}.",
GetType().Name, GrainId.ToDetailedString(), Silo, QueueId.ToStringWithHashCode());
// Remove cast once we cleanup
queueAdapter = qAdapter;
streamFailureHandler = failureHandler;
lastTimeCleanedPubSubCache = DateTime.UtcNow;
try
{
receiver = queueAdapter.CreateReceiver(QueueId);
}
catch (Exception exc)
{
logger.Error(ErrorCode.PersistentStreamPullingAgent_02, "Exception while calling IQueueAdapter.CreateNewReceiver.", exc);
throw;
}
try
{
if (queueAdapterCache != null)
{
queueCache = queueAdapterCache.CreateQueueCache(QueueId);
}
}
catch (Exception exc)
{
logger.Error(ErrorCode.PersistentStreamPullingAgent_23, "Exception while calling IQueueAdapterCache.CreateQueueCache.", exc);
throw;
}
try
{
receiverInitTask = OrleansTaskExtentions.SafeExecute(() => receiver.Initialize(config.InitQueueTimeout))
.LogException(logger, ErrorCode.PersistentStreamPullingAgent_03, $"QueueAdapterReceiver {QueueId.ToStringWithHashCode()} failed to Initialize.");
receiverInitTask.Ignore();
}
catch
{
// Just ignore this exception and proceed as if Initialize has succeeded.
// We already logged individual exceptions for individual calls to Initialize. No need to log again.
}
// Setup a reader for a new receiver.
// Even if the receiver failed to initialise, treat it as OK and start pumping it. It's receiver responsibility to retry initialization.
var randomTimerOffset = safeRandom.NextTimeSpan(config.GetQueueMsgsTimerPeriod);
timer = RegisterTimer(AsyncTimerCallback, QueueId, randomTimerOffset, config.GetQueueMsgsTimerPeriod);
logger.Info((int)ErrorCode.PersistentStreamPullingAgent_04, "Taking queue {0} under my responsibility.", QueueId.ToStringWithHashCode());
}
public IQueueAdapterCache GetQueueAdapterCache()
{
return _adapterCache ?? (_adapterCache = new MemoryCache.MySimpleQueueAdapterCache(this, _logger));
}
public void Init(IProviderConfiguration config, string providerName, Logger logger, IServiceProvider serviceProvider)
{
if (config == null) throw new ArgumentNullException(nameof(config));
if (!config.Properties.TryGetValue(RABBITMQ_CONNECTION_STRING, out _dataConnectionString))
throw new ArgumentException($"{RABBITMQ_CONNECTION_STRING} property not set");
if (!config.Properties.TryGetValue(DEPLOYMENT_ID, out _deploymentId))
throw new ArgumentException($"{DEPLOYMENT_ID} property not set");
string cacheSizeString;
_cacheSize = DEFAULT_CACHE_SIZE;
if (config.Properties.TryGetValue(CACHE_SIZE_PARAM, out cacheSizeString))
if (!int.TryParse(cacheSizeString, out _cacheSize))
throw new ArgumentException($"{CACHE_SIZE_PARAM} invalid. Must be int");
string numQueuesString;
_numQueues = DEFAULT_NUM_QUEUES;
if (config.Properties.TryGetValue(NUM_QUEUES_PARAM, out numQueuesString))
if (!int.TryParse(numQueuesString, out _numQueues))
throw new ArgumentException($"{NUM_QUEUES_PARAM} invalid. Must be int");
_providerName = providerName;
_streamQueueMapper = new HashRingBasedStreamQueueMapper(_numQueues, providerName);
_adapterCache = new SimpleQueueAdapterCache(_cacheSize, logger);
}
