public async Task Init(string name, IProviderRuntime providerUtilitiesManager, IProviderConfiguration config)
{
if (!stateManager.PresetState(ProviderState.Initialized))
{
return;
}
if (String.IsNullOrEmpty(name))
{
throw new ArgumentNullException("name");
}
if (providerUtilitiesManager == null)
{
throw new ArgumentNullException("providerUtilitiesManager");
}
if (config == null)
{
throw new ArgumentNullException("config");
}
Name = name;
providerRuntime = (IStreamProviderRuntime)providerUtilitiesManager;
logger = providerRuntime.GetLogger(this.GetType().Name);
adapterFactory = new TAdapterFactory();
// Temporary change, but we need GrainFactory inside ServiceProvider for now,
// so will change it back as soon as we have an action item to add GrainFactory to ServiceProvider.
adapterFactory.Init(config, Name, logger, new GrainFactoryServiceProvider(providerRuntime));
queueAdapter = await adapterFactory.CreateAdapter();
myConfig = new PersistentStreamProviderConfig(config);
this.providerConfig = config;
this.serializationManager = this.providerRuntime.ServiceProvider.GetRequiredService <SerializationManager>();
this.runtimeClient = this.providerRuntime.ServiceProvider.GetRequiredService <IRuntimeClient>();
if (this.myConfig.PubSubType == StreamPubSubType.ExplicitGrainBasedAndImplicit ||
this.myConfig.PubSubType == StreamPubSubType.ExplicitGrainBasedOnly)
{
this.streamSubscriptionManager = this.providerRuntime.ServiceProvider
.GetService <IStreamSubscriptionManagerAdmin>().GetStreamSubscriptionManager(StreamSubscriptionManagerType.ExplicitSubscribeOnly);
}
string startup;
if (config.Properties.TryGetValue(StartupStatePropertyName, out startup))
{
if (!Enum.TryParse(startup, true, out startupState))
{
throw new ArgumentException(
String.Format("Unsupported value '{0}' for configuration parameter {1} of stream provider {2}.", startup, StartupStatePropertyName, config.Name));
}
}
else
{
startupState = StartupStateDefaultValue;
}
logger.Info("Initialized PersistentStreamProvider<{0}> with name {1}, Adapter {2} and config {3}, {4} = {5}.",
typeof(TAdapterFactory).Name,
Name,
queueAdapter.Name,
myConfig,
StartupStatePropertyName, startupState);
stateManager.CommitState();
}
private static PersistentStreamProviderConfig CreateConfigWithCustomBalancerType()
{
var config = new PersistentStreamProviderConfig();
config.BalancerType = QueueBalancerType;
return(config);
}
public async Task Init(string name, IProviderRuntime providerUtilitiesManager, IProviderConfiguration config)
{
if (!stateManager.PresetState(ProviderState.Initialized))
{
return;
}
if (String.IsNullOrEmpty(name))
{
throw new ArgumentNullException("name");
}
if (providerUtilitiesManager == null)
{
throw new ArgumentNullException("providerUtilitiesManager");
}
if (config == null)
{
throw new ArgumentNullException("config");
}
Name = name;
providerRuntime = (IStreamProviderRuntime)providerUtilitiesManager;
logger = providerRuntime.GetLogger(this.GetType().Name);
adapterFactory = new TAdapterFactory();
adapterFactory.Init(config, Name, logger, providerRuntime.ServiceProvider);
queueAdapter = await adapterFactory.CreateAdapter();
myConfig = new PersistentStreamProviderConfig(config);
string startup;
if (config.Properties.TryGetValue(StartupStatePropertyName, out startup))
{
if (!Enum.TryParse(startup, true, out startupState))
{
throw new ArgumentException(
String.Format("Unsupported value '{0}' for configuration parameter {1} of stream provider {2}.", startup, StartupStatePropertyName, config.Name));
}
}
else
{
startupState = StartupStateDefaultValue;
}
logger.Info("Initialized PersistentStreamProvider<{0}> with name {1}, Adapter {2} and config {3}, {4} = {5}.",
typeof(TAdapterFactory).Name,
Name,
queueAdapter.Name,
myConfig,
StartupStatePropertyName, startupState);
stateManager.CommitState();
}
/// <summary>
/// Adds a stream provider of type <see cref="AzureQueueStreamProvider"/>.
/// </summary>
/// <param name="config">The cluster configuration object to add provider to.</param>
/// <param name="providerName">The provider name</param>
/// <param name="connectionString">The azure storage connection string. If none is provided, it will use the same as in the Globals configuration.</param>
/// <param name="numberOfQueues">The number of queues to use as partitions.</param>
/// <param name="deploymentId">The deployment ID used for partitioning. If none is specified, the provider will use the same DeploymentId as the Cluster.</param>
/// <param name="cacheSize">The cache size.</param>
/// <param name="startupState">The startup state of the persistent stream provider.</param>
/// <param name="persistentStreamProviderConfig">Settings related to all persistent stream providers.</param>
public static void AddAzureQueueStreamProvider(
this ClientConfiguration config,
string providerName,
string connectionString = null,
int numberOfQueues = AzureQueueAdapterFactory.NumQueuesDefaultValue,
string deploymentId = null,
int cacheSize = AzureQueueAdapterFactory.CacheSizeDefaultValue,
PersistentStreamProviderState startupState = AzureQueueStreamProvider.StartupStateDefaultValue,
PersistentStreamProviderConfig persistentStreamProviderConfig = null)
{
connectionString = GetConnectionString(connectionString, config);
deploymentId = deploymentId ?? config.DeploymentId;
var properties = GetAzureQueueStreamProviderProperties(providerName, connectionString, numberOfQueues, deploymentId, cacheSize, startupState, persistentStreamProviderConfig);
config.RegisterStreamProvider <AzureQueueStreamProvider>(providerName, properties);
}
/// <summary>
/// Adds a stream provider of type <see cref="AzureQueueStreamProviderV2"/>.
/// </summary>
/// <param name="config">The cluster configuration object to add provider to.</param>
/// <param name="providerName">The provider name</param>
/// <param name="connectionString">The azure storage connection string. If none is provided, it will use the same as in the Globals configuration.</param>
/// <param name="numberOfQueues">The number of queues to use as partitions.</param>
/// <param name="clusterId">The ClusterId used for partitioning. If none is specified, the provider will use the same ClusterId as the Cluster.</param>
/// <param name="cacheSize">The cache size.</param>
/// <param name="startupState">The startup state of the persistent stream provider.</param>
/// <param name="persistentStreamProviderConfig">Settings related to all persistent stream providers.</param>
public static void AddAzureQueueStreamProviderV2(
this ClusterConfiguration config,
string providerName,
string connectionString = null,
int numberOfQueues = AzureQueueAdapterConstants.NumQueuesDefaultValue,
string clusterId = null,
int cacheSize = AzureQueueAdapterConstants.CacheSizeDefaultValue,
#pragma warning disable 618
PersistentStreamProviderState startupState = AzureQueueStreamProvider.StartupStateDefaultValue,
#pragma warning restore 618
PersistentStreamProviderConfig persistentStreamProviderConfig = null)
{
connectionString = GetConnectionString(connectionString, config);
clusterId = clusterId ?? config.Globals.ClusterId;
var properties = GetAzureQueueStreamProviderProperties(providerName, connectionString, numberOfQueues, clusterId, cacheSize, startupState, persistentStreamProviderConfig);
config.Globals.RegisterStreamProvider <AzureQueueStreamProviderV2>(providerName, properties);
}
public async Task <IPersistentStreamPullingManager> InitializePullingAgents(
string streamProviderName,
IQueueAdapterFactory adapterFactory,
IQueueAdapter queueAdapter,
PersistentStreamProviderConfig config)
{
IStreamQueueBalancer queueBalancer = StreamQueueBalancerFactory.Create(config.BalancerType, streamProviderName, this.siloStatusOracle, this.siloDetails.ClusterConfig, this, adapterFactory.GetStreamQueueMapper(), config.SiloMaturityPeriod);
var managerId = GrainId.NewSystemTargetGrainIdByTypeCode(Constants.PULLING_AGENTS_MANAGER_SYSTEM_TARGET_TYPE_CODE);
var manager = new PersistentStreamPullingManager(managerId, streamProviderName, this, this.PubSub(config.PubSubType), adapterFactory, queueBalancer, config);
this.RegisterSystemTarget(manager);
// Init the manager only after it was registered locally.
var pullingAgentManager = manager.AsReference <IPersistentStreamPullingManager>();
// Need to call it as a grain reference though.
await pullingAgentManager.Initialize(queueAdapter.AsImmutable());
return(pullingAgentManager);
}
public override TestCluster CreateTestCluster()
{
// Define a cluster of 4, but deploy ony 2 to start.
var options = new TestClusterOptions(4);
options.ClusterConfiguration.AddMemoryStorageProvider("PubSubStore");
var persistentStreamProviderConfig = new PersistentStreamProviderConfig
{
SiloMaturityPeriod = SILO_IMMATURE_PERIOD,
BalancerType = StreamQueueBalancerType.DynamicClusterConfigDeploymentBalancer,
};
options.ClusterConfiguration.AddAzureQueueStreamProvider(adapterName, persistentStreamProviderConfig: persistentStreamProviderConfig);
options.ClientConfiguration.Gateways = options.ClientConfiguration.Gateways.Take(1).ToList();
var host = new TestCluster(options);
host.Deploy(new[] { Silo.PrimarySiloName, "Secondary_1" });
return(host);
}
protected override void ConfigureTestCluster(TestClusterBuilder builder)
{
TestUtils.CheckForAzureStorage();
// Define a cluster of 4, but 2 will be stopped.
builder.Options.InitialSilosCount = 4;
builder.ConfigureLegacyConfiguration(legacy =>
{
legacy.ClusterConfiguration.AddMemoryStorageProvider("PubSubStore");
var persistentStreamProviderConfig = new PersistentStreamProviderConfig
{
SiloMaturityPeriod = SILO_IMMATURE_PERIOD,
BalancerType = StreamQueueBalancerType.DynamicClusterConfigDeploymentBalancer,
};
legacy.ClusterConfiguration.AddAzureQueueStreamProvider(adapterName, persistentStreamProviderConfig: persistentStreamProviderConfig);
legacy.ClientConfiguration.Gateways = legacy.ClientConfiguration.Gateways.Take(1).ToList();
});
}
/// <summary>
/// Adds a stream provider of type <see cref="AzureQueueStreamProvider"/>.
/// </summary>
/// <param name="config">The cluster configuration object to add provider to.</param>
/// <param name="providerName">The provider name</param>
/// <param name="connectionString">The azure storage connection string. If none is provided, it will use the same as in the Globals configuration.</param>
/// <param name="numberOfQueues">The number of queues to use as partitions.</param>
/// <param name="deploymentId">The deployment ID used for partitioning. If none is specified, the provider will use the same DeploymentId as the Cluster.</param>
/// <param name="cacheSize">The cache size.</param>
/// <param name="startupState">The startup state of the persistent stream provider.</param>
/// <param name="persistentStreamProviderConfig">Settings related to all persistent stream providers.</param>
public static void AddAzureQueueStreamProvider(
this ClusterConfiguration config,
string providerName,
string connectionString = null,
int numberOfQueues = AzureQueueAdapterConstants.NumQueuesDefaultValue,
string deploymentId = null,
int cacheSize = AzureQueueAdapterConstants.CacheSizeDefaultValue,
#pragma warning disable CS0618 // Type or member is obsolete
PersistentStreamProviderState startupState = AzureQueueStreamProvider.StartupStateDefaultValue,
#pragma warning restore CS0618 // Type or member is obsolete
PersistentStreamProviderConfig persistentStreamProviderConfig = null)
{
connectionString = GetConnectionString(connectionString, config);
deploymentId = deploymentId ?? config.Globals.DeploymentId;
var properties = GetAzureQueueStreamProviderProperties(providerName, connectionString, numberOfQueues, deploymentId, cacheSize, startupState, persistentStreamProviderConfig);
#pragma warning disable 618
config.Globals.RegisterStreamProvider <AzureQueueStreamProvider>(providerName, properties);
#pragma warning restore 618
}
private IStreamQueueBalancer CreateQueueBalancer(PersistentStreamProviderConfig config, string streamProviderName)
{
//default type is ConsistentRingBalancer
if (config.BalancerType == null)
{
config.BalancerType = StreamQueueBalancerType.ConsistentRingBalancer;
}
try
{
var balancer = this.ServiceProvider.GetRequiredService(config.BalancerType) as IStreamQueueBalancer;
if (balancer == null)
{
throw new ArgumentOutOfRangeException("balancerType", $"Configured BalancerType isn't a type which implements IStreamQueueBalancer. BalancerType: {config.BalancerType}, StreamProvider: {streamProviderName}");
}
return(balancer);
}
catch (Exception e)
{
string error = $"Unsupported balancerType for stream provider. BalancerType: {config.BalancerType}, StreamProvider: {streamProviderName}, Exception: {e}";
throw new ArgumentOutOfRangeException("balancerType", error);
}
}
private static Dictionary <string, string> GetAzureQueueStreamProviderProperties(string providerName, string connectionString, int numberOfQueues, string deploymentId, int cacheSize, PersistentStreamProviderState startupState, PersistentStreamProviderConfig persistentStreamProviderConfig)
{
if (string.IsNullOrWhiteSpace(providerName))
{
throw new ArgumentNullException(nameof(providerName));
}
if (numberOfQueues < 1)
{
throw new ArgumentOutOfRangeException(nameof(numberOfQueues));
}
var properties = new Dictionary <string, string>
{
{ AzureQueueAdapterFactory.DataConnectionStringPropertyName, connectionString },
{ AzureQueueAdapterFactory.NumQueuesPropertyName, numberOfQueues.ToString() },
{ AzureQueueAdapterFactory.DeploymentIdPropertyName, deploymentId },
{ SimpleQueueAdapterCache.CacheSizePropertyName, cacheSize.ToString() },
{ AzureQueueStreamProvider.StartupStatePropertyName, startupState.ToString() },
};
persistentStreamProviderConfig?.WriteProperties(properties);
return(properties);
}
