internal PersistentStreamPullingAgent(
GrainId id,
string strProviderName,
IStreamProviderRuntime runtime,
IStreamPubSub streamPubSub,
QueueId queueId,
PersistentStreamProviderConfig config)
: base(id, runtime.ExecutingSiloAddress, true)
{
if (runtime == null) throw new ArgumentNullException("runtime", "PersistentStreamPullingAgent: runtime reference should not be null");
if (strProviderName == null) throw new ArgumentNullException("runtime", "PersistentStreamPullingAgent: strProviderName should not be null");
QueueId = queueId;
streamProviderName = strProviderName;
providerRuntime = runtime;
pubSub = streamPubSub;
pubSubCache = new Dictionary<StreamId, StreamConsumerCollection>();
safeRandom = new SafeRandom();
this.config = config;
numMessages = 0;
logger = providerRuntime.GetLogger(GrainId + "-" + streamProviderName);
logger.Info((int)ErrorCode.PersistentStreamPullingAgent_01,
"Created {0} {1} for Stream Provider {2} on silo {3} for Queue {4}.",
GetType().Name, GrainId.ToDetailedString(), streamProviderName, Silo, QueueId.ToStringWithHashCode());
string statUniquePostfix = strProviderName + "." + QueueId;
numReadMessagesCounter = CounterStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_READ_MESSAGES, statUniquePostfix));
numSentMessagesCounter = CounterStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_SENT_MESSAGES, statUniquePostfix));
IntValueStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_PUBSUB_CACHE_SIZE, statUniquePostfix), () => pubSubCache.Count);
// TODO: move queue cache size statistics tracking into queue cache implementation once Telemetry APIs and LogStatistics have been reconciled.
//IntValueStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_QUEUE_CACHE_SIZE, statUniquePostfix), () => queueCache != null ? queueCache.Size : 0);
}
internal PersistentStreamPullingAgent(
GrainId id,
string strProviderName,
IStreamProviderRuntime runtime,
IStreamPubSub streamPubSub,
QueueId queueId,
PersistentStreamProviderConfig config)
: base(id, runtime.ExecutingSiloAddress, true)
{
if (runtime == null)
{
throw new ArgumentNullException("runtime", "PersistentStreamPullingAgent: runtime reference should not be null");
}
if (strProviderName == null)
{
throw new ArgumentNullException("runtime", "PersistentStreamPullingAgent: strProviderName should not be null");
}
QueueId = queueId;
streamProviderName = strProviderName;
pubSub = streamPubSub;
pubSubCache = new Dictionary <StreamId, StreamConsumerCollection>();
safeRandom = new SafeRandom();
this.config = config;
numMessages = 0;
logger = runtime.GetLogger(((ISystemTargetBase)this).GrainId + "-" + streamProviderName);
logger.Info(ErrorCode.PersistentStreamPullingAgent_01,
"Created {0} {1} for Stream Provider {2} on silo {3} for Queue {4}.",
GetType().Name, ((ISystemTargetBase)this).GrainId.ToDetailedString(), streamProviderName, Silo, QueueId.ToStringWithHashCode());
numReadMessagesCounter = CounterStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_READ_MESSAGES, StatisticUniquePostfix));
numSentMessagesCounter = CounterStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_SENT_MESSAGES, StatisticUniquePostfix));
// TODO: move queue cache size statistics tracking into queue cache implementation once Telemetry APIs and LogStatistics have been reconciled.
//IntValueStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_QUEUE_CACHE_SIZE, statUniquePostfix), () => queueCache != null ? queueCache.Size : 0);
}
internal PersistentStreamPullingAgent(
GrainId id,
string strProviderName,
IStreamProviderRuntime runtime,
IStreamPubSub streamPubSub,
QueueId queueId,
PersistentStreamProviderConfig config)
: base(id, runtime.ExecutingSiloAddress, true)
{
if (runtime == null) throw new ArgumentNullException("runtime", "PersistentStreamPullingAgent: runtime reference should not be null");
if (strProviderName == null) throw new ArgumentNullException("runtime", "PersistentStreamPullingAgent: strProviderName should not be null");
QueueId = queueId;
streamProviderName = strProviderName;
providerRuntime = runtime;
pubSub = streamPubSub;
pubSubCache = new Dictionary<StreamId, StreamConsumerCollection>();
safeRandom = new SafeRandom();
this.config = config;
numMessages = 0;
logger = providerRuntime.GetLogger(GrainId + "-" + streamProviderName);
logger.Info((int)ErrorCode.PersistentStreamPullingAgent_01,
"Created {0} {1} for Stream Provider {2} on silo {3} for Queue {4}.",
GetType().Name, GrainId.ToDetailedString(), streamProviderName, Silo, QueueId.ToStringWithHashCode());
string statUniquePostfix = strProviderName + "." + QueueId;
numReadMessagesCounter = CounterStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_READ_MESSAGES, statUniquePostfix));
numSentMessagesCounter = CounterStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_SENT_MESSAGES, statUniquePostfix));
IntValueStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_PUBSUB_CACHE_SIZE, statUniquePostfix), () => pubSubCache.Count);
IntValueStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_QUEUE_CACHE_SIZE, statUniquePostfix), () => queueCache !=null ? queueCache.Size : 0);
}
internal PersistentStreamPullingManager(
GrainId id,
string strProviderName,
IStreamProviderRuntime runtime,
IStreamPubSub streamPubSub,
IQueueAdapterFactory adapterFactory,
IStreamQueueBalancer streamQueueBalancer,
PersistentStreamProviderConfig config,
IProviderConfiguration providerConfig,
ILoggerFactory loggerFactory)
: base(id, runtime.ExecutingSiloAddress, loggerFactory)
{
if (string.IsNullOrWhiteSpace(strProviderName))
{
throw new ArgumentNullException("strProviderName");
}
if (runtime == null)
{
throw new ArgumentNullException("runtime", "IStreamProviderRuntime runtime reference should not be null");
}
if (streamPubSub == null)
{
throw new ArgumentNullException("streamPubSub", "StreamPubSub reference should not be null");
}
if (streamQueueBalancer == null)
{
throw new ArgumentNullException("streamQueueBalancer", "IStreamQueueBalancer streamQueueBalancer reference should not be null");
}
queuesToAgentsMap = new Dictionary <QueueId, PersistentStreamPullingAgent>();
streamProviderName = strProviderName;
providerRuntime = runtime;
pubSub = streamPubSub;
this.config = config;
this.providerConfig = providerConfig;
nonReentrancyGuarantor = new AsyncSerialExecutor();
latestRingNotificationSequenceNumber = 0;
latestCommandNumber = 0;
queueBalancer = streamQueueBalancer;
this.adapterFactory = adapterFactory;
queueAdapterCache = adapterFactory.GetQueueAdapterCache();
logger = loggerFactory.CreateLogger($"{GetType().FullName}-{streamProviderName}");
Log(ErrorCode.PersistentStreamPullingManager_01, "Created {0} for Stream Provider {1}.", GetType().Name, streamProviderName);
this.loggerFactory = loggerFactory;
IntValueStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_PULLING_AGENTS, strProviderName), () => queuesToAgentsMap.Count);
}
internal PersistentStreamPullingManager(
GrainId id,
string strProviderName,
IStreamProviderRuntime runtime,
IStreamPubSub streamPubSub,
IQueueAdapterFactory adapterFactory,
IStreamQueueBalancer streamQueueBalancer,
PersistentStreamProviderConfig config)
: base(id, runtime.ExecutingSiloAddress)
{
if (string.IsNullOrWhiteSpace(strProviderName))
{
throw new ArgumentNullException("strProviderName");
}
if (runtime == null)
{
throw new ArgumentNullException("runtime", "IStreamProviderRuntime runtime reference should not be null");
}
if (streamPubSub == null)
{
throw new ArgumentNullException("streamPubSub", "StreamPubSub reference should not be null");
}
if (streamQueueBalancer == null)
{
throw new ArgumentNullException("streamQueueBalancer", "IStreamQueueBalancer streamQueueBalancer reference should not be null");
}
queuesToAgentsMap = new Dictionary<QueueId, PersistentStreamPullingAgent>();
streamProviderName = strProviderName;
providerRuntime = runtime;
pubSub = streamPubSub;
this.config = config;
nonReentrancyGuarantor = new AsyncSerialExecutor();
latestRingNotificationSequenceNumber = 0;
latestCommandNumber = 0;
queueBalancer = streamQueueBalancer;
this.adapterFactory = adapterFactory;
queueAdapterCache = adapterFactory.GetQueueAdapterCache();
logger = TraceLogger.GetLogger(GetType().Name + "-" + streamProviderName, TraceLogger.LoggerType.Provider);
Log(ErrorCode.PersistentStreamPullingManager_01, "Created {0} for Stream Provider {1}.", GetType().Name, streamProviderName);
IntValueStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_PULLING_AGENTS, strProviderName), () => queuesToAgentsMap.Count);
queuePrintTimer = base.RegisterTimer(AsyncTimerCallback, null, QUEUES_PRINT_PERIOD, QUEUES_PRINT_PERIOD);
}
internal PersistentStreamPullingManager(
GrainId id,
string strProviderName,
IStreamProviderRuntime runtime,
IStreamPubSub streamPubSub,
IQueueAdapterFactory adapterFactory,
IStreamQueueBalancer streamQueueBalancer,
PersistentStreamProviderConfig config)
: base(id, runtime.ExecutingSiloAddress)
{
if (string.IsNullOrWhiteSpace(strProviderName))
{
throw new ArgumentNullException("strProviderName");
}
if (runtime == null)
{
throw new ArgumentNullException("runtime", "IStreamProviderRuntime runtime reference should not be null");
}
if (streamPubSub == null)
{
throw new ArgumentNullException("streamPubSub", "StreamPubSub reference should not be null");
}
if (streamQueueBalancer == null)
{
throw new ArgumentNullException("streamQueueBalancer", "IStreamQueueBalancer streamQueueBalancer reference should not be null");
}
queuesToAgentsMap = new Dictionary <QueueId, PersistentStreamPullingAgent>();
streamProviderName = strProviderName;
providerRuntime = runtime;
pubSub = streamPubSub;
this.config = config;
nonReentrancyGuarantor = new AsyncSerialExecutor();
latestRingNotificationSequenceNumber = 0;
latestCommandNumber = 0;
queueBalancer = streamQueueBalancer;
this.adapterFactory = adapterFactory;
queueAdapterCache = adapterFactory.GetQueueAdapterCache();
logger = TraceLogger.GetLogger(GetType().Name + "-" + streamProviderName, TraceLogger.LoggerType.Provider);
logger.Info((int)ErrorCode.PersistentStreamPullingManager_01, "Created {0} for Stream Provider {1}.", GetType().Name, streamProviderName);
IntValueStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_PULLING_AGENTS, strProviderName), () => queuesToAgentsMap.Count);
queuePrintTimer = base.RegisterTimer(AsyncTimerCallback, null, QUEUES_PRINT_PERIOD, QUEUES_PRINT_PERIOD);
}
public override TestCluster CreateTestCluster()
{
// Define a cluster of 4, but deploy ony 2 to start.
var options = new TestClusterOptions(4);
options.ClusterConfiguration.AddMemoryStorageProvider("PubSubStore");
var persistentStreamProviderConfig = new PersistentStreamProviderConfig
{
SiloMaturityPeriod = SILO_IMMATURE_PERIOD,
BalancerType = StreamQueueBalancerType.DynamicClusterConfigDeploymentBalancer,
};
options.ClusterConfiguration.AddAzureQueueStreamProvider(adapterName, persistentStreamProviderConfig: persistentStreamProviderConfig);
options.ClientConfiguration.Gateways = options.ClientConfiguration.Gateways.Take(1).ToList();
var host = new TestCluster(options);
host.Deploy(new[] { Silo.PrimarySiloName, "Secondary_1" });
return host;
}
/// <summary>
/// Adds a stream provider of type <see cref="AzureQueueStreamProvider"/>.
/// </summary>
/// <param name="config">The cluster configuration object to add provider to.</param>
/// <param name="providerName">The provider name</param>
/// <param name="connectionString">The azure storage connection string. If none is provided, it will use the same as in the Globals configuration.</param>
/// <param name="numberOfQueues">The number of queues to use as partitions.</param>
/// <param name="deploymentId">The deployment ID used for partitioning. If none is specified, the provider will use the same DeploymentId as the Cluster.</param>
/// <param name="cacheSize">The cache size.</param>
/// <param name="startupState">The startup state of the persistent stream provider.</param>
/// <param name="persistentStreamProviderConfig">Settings related to all persistent stream providers.</param>
public static void AddAzureQueueStreamProvider(
this ClusterConfiguration config,
string providerName,
string connectionString = null,
int numberOfQueues = AzureQueueAdapterFactory.NumQueuesDefaultValue,
string deploymentId = null,
int cacheSize = AzureQueueAdapterFactory.CacheSizeDefaultValue,
PersistentStreamProviderState startupState = AzureQueueStreamProvider.StartupStateDefaultValue,
PersistentStreamProviderConfig persistentStreamProviderConfig = null)
{
connectionString = GetConnectionString(connectionString, config);
deploymentId = deploymentId ?? config.Globals.DeploymentId;
var properties = GetAzureQueueStreamProviderProperties(providerName, connectionString, numberOfQueues, deploymentId, cacheSize, startupState, persistentStreamProviderConfig);
config.Globals.RegisterStreamProvider<AzureQueueStreamProvider>(providerName, properties);
}
private static Dictionary<string, string> GetAzureQueueStreamProviderProperties(string providerName, string connectionString, int numberOfQueues, string deploymentId, int cacheSize, PersistentStreamProviderState startupState, PersistentStreamProviderConfig persistentStreamProviderConfig)
{
if (string.IsNullOrWhiteSpace(providerName)) throw new ArgumentNullException(nameof(providerName));
if (numberOfQueues < 1) throw new ArgumentOutOfRangeException(nameof(numberOfQueues));
var properties = new Dictionary<string, string>
{
{ AzureQueueAdapterFactory.DataConnectionStringPropertyName, connectionString },
{ AzureQueueAdapterFactory.NumQueuesPropertyName, numberOfQueues.ToString() },
{ AzureQueueAdapterFactory.DeploymentIdPropertyName, deploymentId },
{ SimpleQueueAdapterCache.CacheSizePropertyName, cacheSize.ToString() },
{ AzureQueueStreamProvider.StartupStatePropertyName, startupState.ToString() },
};
persistentStreamProviderConfig?.WriteProperties(properties);
return properties;
}