internal PersistentStreamPullingAgent(
GrainId id,
string strProviderName,
IStreamProviderRuntime runtime,
IStreamPubSub streamPubSub,
QueueId queueId,
PersistentStreamProviderConfig config)
: base(id, runtime.ExecutingSiloAddress, true)
{
if (runtime == null)
{
throw new ArgumentNullException("runtime", "PersistentStreamPullingAgent: runtime reference should not be null");
}
if (strProviderName == null)
{
throw new ArgumentNullException("runtime", "PersistentStreamPullingAgent: strProviderName should not be null");
}
QueueId = queueId;
streamProviderName = strProviderName;
pubSub = streamPubSub;
pubSubCache = new Dictionary <StreamId, StreamConsumerCollection>();
safeRandom = new SafeRandom();
this.config = config;
numMessages = 0;
logger = runtime.GetLogger(((ISystemTargetBase)this).GrainId + "-" + streamProviderName);
logger.Info(ErrorCode.PersistentStreamPullingAgent_01,
"Created {0} {1} for Stream Provider {2} on silo {3} for Queue {4}.",
GetType().Name, ((ISystemTargetBase)this).GrainId.ToDetailedString(), streamProviderName, Silo, QueueId.ToStringWithHashCode());
numReadMessagesCounter = CounterStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_READ_MESSAGES, StatisticUniquePostfix));
numSentMessagesCounter = CounterStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_SENT_MESSAGES, StatisticUniquePostfix));
// TODO: move queue cache size statistics tracking into queue cache implementation once Telemetry APIs and LogStatistics have been reconciled.
//IntValueStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_QUEUE_CACHE_SIZE, statUniquePostfix), () => queueCache != null ? queueCache.Size : 0);
}
public StreamSubscriptionManagerAdmin(IStreamProviderRuntime providerRuntime)
{
// using ExplicitGrainBasedAndImplicit pubsub here, so if StreamSubscriptionManager.Add(Remove)Subscription called on a implicit subscribed
// consumer grain, its subscription will be handled by ImplicitPubsub, and will not be messed into GrainBasedPubsub
_explicitStreamSubscriptionManager = new StreamSubscriptionManager(providerRuntime.PubSub(StreamPubSubType.ExplicitGrainBasedAndImplicit),
StreamSubscriptionManagerType.ExplicitSubscribeOnly);
}
public StreamConsumer(StreamImpl <T> stream, string streamProviderName, IStreamProviderRuntime providerUtilities, IStreamPubSub pubSub, bool isRewindable)
{
if (stream == null)
{
throw new ArgumentNullException("stream");
}
if (providerUtilities == null)
{
throw new ArgumentNullException("providerUtilities");
}
if (pubSub == null)
{
throw new ArgumentNullException("pubSub");
}
logger = TraceLogger.GetLogger(string.Format("StreamConsumer<{0}>-{1}", typeof(T).Name, stream), TraceLogger.LoggerType.Runtime);
this.stream = stream;
this.streamProviderName = streamProviderName;
providerRuntime = providerUtilities;
this.pubSub = pubSub;
IsRewindable = isRewindable;
myExtension = null;
myGrainReference = null;
bindExtLock = new AsyncLock();
}
internal StreamConsumerExtension(IStreamProviderRuntime providerRt, IStreamSubscriptionObserver streamSubscriptionObserver = null)
{
this.streamSubscriptionObserver = streamSubscriptionObserver;
providerRuntime = providerRt;
allStreamObservers = new ConcurrentDictionary <GuidId, IStreamSubscriptionHandle>();
logger = providerRt.ServiceProvider.GetRequiredService <ILogger <StreamConsumerExtension> >();
}
internal PersistentStreamPullingAgent(
GrainId id,
string strProviderName,
IStreamProviderRuntime runtime,
QueueId queueId,
TimeSpan queueGetPeriod,
TimeSpan initQueueTimeout,
TimeSpan maxDeliveryTime)
: base(id, runtime.ExecutingSiloAddress, true)
{
if (runtime == null) throw new ArgumentNullException("runtime", "PersistentStreamPullingAgent: runtime reference should not be null");
if (strProviderName == null) throw new ArgumentNullException("runtime", "PersistentStreamPullingAgent: strProviderName should not be null");
QueueId = queueId;
streamProviderName = strProviderName;
providerRuntime = runtime;
pubSub = runtime.PubSub(StreamPubSubType.GrainBased);
pubSubCache = new Dictionary<StreamId, StreamConsumerCollection>();
safeRandom = new SafeRandom();
this.queueGetPeriod = queueGetPeriod;
this.initQueueTimeout = initQueueTimeout;
this.maxDeliveryTime = maxDeliveryTime;
numMessages = 0;
logger = providerRuntime.GetLogger(GrainId + "-" + streamProviderName);
logger.Info((int)ErrorCode.PersistentStreamPullingAgent_01,
"Created {0} {1} for Stream Provider {2} on silo {3} for Queue {4}.",
GetType().Name, GrainId.ToDetailedString(), streamProviderName, Silo, QueueId.ToStringWithHashCode());
numReadMessagesCounter = CounterStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_READ_MESSAGES, strProviderName));
numSentMessagesCounter = CounterStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_SENT_MESSAGES, strProviderName));
}
public PersistentStreamProvider(string name, PersistentStreamOptions options, IProviderRuntime runtime, SerializationManager serializationManager, ILogger <PersistentStreamProvider> logger)
{
if (String.IsNullOrEmpty(name))
{
throw new ArgumentNullException(nameof(name));
}
if (options == null)
{
throw new ArgumentNullException(nameof(options));
}
if (runtime == null)
{
throw new ArgumentNullException(nameof(runtime));
}
if (serializationManager == null)
{
throw new ArgumentNullException(nameof(serializationManager));
}
if (logger == null)
{
throw new ArgumentNullException(nameof(logger));
}
this.Name = name;
this.options = options;
this.runtime = runtime.ServiceProvider.GetRequiredService <IStreamProviderRuntime>();
this.runtimeClient = runtime.ServiceProvider.GetRequiredService <IRuntimeClient>();
this.serializationManager = serializationManager;
this.logger = logger;
}
internal SimpleMessageStreamProducerExtension(IStreamProviderRuntime providerRt, bool fireAndForget)
{
providerRuntime = providerRt;
fireAndForgetDelivery = fireAndForget;
remoteConsumers = new Dictionary<StreamId, StreamConsumerExtensionCollection>();
logger = providerRuntime.GetLogger(GetType().Name);
}
public async Task Init(string name, IProviderRuntime providerUtilitiesManager, IProviderConfiguration config)
{
if (!stateManager.PresetState(ProviderState.Initialized))
{
return;
}
if (String.IsNullOrEmpty(name))
{
throw new ArgumentNullException("name");
}
if (providerUtilitiesManager == null)
{
throw new ArgumentNullException("providerUtilitiesManager");
}
if (config == null)
{
throw new ArgumentNullException("config");
}
Name = name;
providerRuntime = (IStreamProviderRuntime)providerUtilitiesManager;
logger = providerRuntime.GetLogger(this.GetType().Name);
adapterFactory = new TAdapterFactory();
// Temporary change, but we need GrainFactory inside ServiceProvider for now,
// so will change it back as soon as we have an action item to add GrainFactory to ServiceProvider.
adapterFactory.Init(config, Name, logger, new GrainFactoryServiceProvider(providerRuntime));
queueAdapter = await adapterFactory.CreateAdapter();
myConfig = new PersistentStreamProviderConfig(config);
this.providerConfig = config;
this.serializationManager = this.providerRuntime.ServiceProvider.GetRequiredService <SerializationManager>();
this.runtimeClient = this.providerRuntime.ServiceProvider.GetRequiredService <IRuntimeClient>();
if (this.myConfig.PubSubType == StreamPubSubType.ExplicitGrainBasedAndImplicit ||
this.myConfig.PubSubType == StreamPubSubType.ExplicitGrainBasedOnly)
{
this.streamSubscriptionManager = this.providerRuntime.ServiceProvider
.GetService <IStreamSubscriptionManagerAdmin>().GetStreamSubscriptionManager(StreamSubscriptionManagerType.ExplicitSubscribeOnly);
}
string startup;
if (config.Properties.TryGetValue(StartupStatePropertyName, out startup))
{
if (!Enum.TryParse(startup, true, out startupState))
{
throw new ArgumentException(
String.Format("Unsupported value '{0}' for configuration parameter {1} of stream provider {2}.", startup, StartupStatePropertyName, config.Name));
}
}
else
{
startupState = StartupStateDefaultValue;
}
logger.Info("Initialized PersistentStreamProvider<{0}> with name {1}, Adapter {2} and config {3}, {4} = {5}.",
typeof(TAdapterFactory).Name,
Name,
queueAdapter.Name,
myConfig,
StartupStatePropertyName, startupState);
stateManager.CommitState();
}
/// <summary>
/// Create stream queue balancer by type requested
/// </summary>
/// <param name="balancerType">queue balancer type to create</param>
/// <param name="strProviderName">name of requesting stream provider</param>
/// <param name="runtime">stream provider runtime environment to run in</param>
/// <param name="queueMapper">queue mapper of requesting stream provider</param>
/// <returns>Constructed stream queue balancer</returns>
public static IStreamQueueBalancer Create(
StreamQueueBalancerType balancerType,
string strProviderName,
IStreamProviderRuntime runtime,
IStreamQueueMapper queueMapper)
{
if (string.IsNullOrWhiteSpace(strProviderName))
{
throw new ArgumentNullException("strProviderName");
}
if (runtime == null)
{
throw new ArgumentNullException("runtime");
}
if (queueMapper == null)
{
throw new ArgumentNullException("queueMapper");
}
switch (balancerType)
{
case StreamQueueBalancerType.ConsistenRingBalancer:
{
// Consider: for now re-use the same ConsistentRingProvider with 1 equally devided range. Remove later.
IConsistentRingProviderForGrains ringProvider = runtime.GetConsistentRingProvider(0, 1);
return(new ConsistentRingQueueBalancer(ringProvider, queueMapper));
}
default:
{
string error = string.Format("Unsupported balancerType for stream provider. BalancerType: {0}, StreamProvider: {1}", balancerType, strProviderName);
throw new ArgumentOutOfRangeException("balancerType", error);
}
}
}
internal PersistentStreamPullingAgent(
GrainId id,
string strProviderName,
IStreamProviderRuntime runtime,
IStreamPubSub streamPubSub,
QueueId queueId,
PersistentStreamProviderConfig config)
: base(id, runtime.ExecutingSiloAddress, true)
{
if (runtime == null) throw new ArgumentNullException("runtime", "PersistentStreamPullingAgent: runtime reference should not be null");
if (strProviderName == null) throw new ArgumentNullException("runtime", "PersistentStreamPullingAgent: strProviderName should not be null");
QueueId = queueId;
streamProviderName = strProviderName;
providerRuntime = runtime;
pubSub = streamPubSub;
pubSubCache = new Dictionary<StreamId, StreamConsumerCollection>();
safeRandom = new SafeRandom();
this.config = config;
numMessages = 0;
logger = providerRuntime.GetLogger(GrainId + "-" + streamProviderName);
logger.Info((int)ErrorCode.PersistentStreamPullingAgent_01,
"Created {0} {1} for Stream Provider {2} on silo {3} for Queue {4}.",
GetType().Name, GrainId.ToDetailedString(), streamProviderName, Silo, QueueId.ToStringWithHashCode());
string statUniquePostfix = strProviderName + "." + QueueId;
numReadMessagesCounter = CounterStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_READ_MESSAGES, statUniquePostfix));
numSentMessagesCounter = CounterStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_SENT_MESSAGES, statUniquePostfix));
IntValueStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_PUBSUB_CACHE_SIZE, statUniquePostfix), () => pubSubCache.Count);
IntValueStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_QUEUE_CACHE_SIZE, statUniquePostfix), () => queueCache !=null ? queueCache.Size : 0);
}
public SimpleMessageStreamProvider(
string name,
SimpleMessageStreamProviderOptions options,
IStreamFilter streamFilter,
ILoggerFactory loggerFactory,
IServiceProvider services)
{
this.loggerFactory = loggerFactory;
this.Name = name;
this.logger = loggerFactory.CreateLogger <SimpleMessageStreamProvider>();
this.options = options;
this.streamFilter = streamFilter;
this.providerRuntime = services.GetRequiredService <IStreamProviderRuntime>();
this.runtimeClient = providerRuntime.ServiceProvider.GetService <IRuntimeClient>();
if (this.options.PubSubType == StreamPubSubType.ExplicitGrainBasedAndImplicit ||
this.options.PubSubType == StreamPubSubType.ExplicitGrainBasedOnly)
{
this.streamSubscriptionManager = this.providerRuntime.ServiceProvider
.GetService <IStreamSubscriptionManagerAdmin>()
.GetStreamSubscriptionManager(StreamSubscriptionManagerType.ExplicitSubscribeOnly);
}
logger.Info(
"Initialized SimpleMessageStreamProvider with name {0} and with property FireAndForgetDelivery: {1}, OptimizeForImmutableData: {2} " +
"and PubSubType: {3}", Name, this.options.FireAndForgetDelivery, this.options.OptimizeForImmutableData,
this.options.PubSubType);
}
internal StreamConsumerExtension(IStreamProviderRuntime providerRt, StreamSubscriptionChangeHandler streamSubscriptionChangeHandler = null)
{
this.subscriptionChangeHandler = streamSubscriptionChangeHandler;
providerRuntime = providerRt;
allStreamObservers = new ConcurrentDictionary <GuidId, IStreamSubscriptionHandle>();
logger = providerRuntime.GetLogger(GetType().Name);
}
internal StreamConsumerExtension(IStreamProviderRuntime providerRt, bool isRewindable)
{
providerRuntime = providerRt;
allStreamObservers = new ConcurrentDictionary <GuidId, IStreamSubscriptionHandle>();
this.isRewindable = isRewindable;
logger = providerRuntime.GetLogger(GetType().Name);
}
public StreamConsumer(StreamImpl <T> stream, string streamProviderName, IStreamProviderRuntime runtime, IStreamPubSub pubSub, ILogger logger, bool isRewindable)
{
if (stream == null)
{
throw new ArgumentNullException(nameof(stream));
}
if (runtime == null)
{
throw new ArgumentNullException(nameof(runtime));
}
if (pubSub == null)
{
throw new ArgumentNullException(nameof(pubSub));
}
if (logger == null)
{
throw new ArgumentNullException(nameof(logger));
}
this.logger = logger;
this.stream = stream;
this.streamProviderName = streamProviderName;
this.providerRuntime = runtime;
this.pubSub = pubSub;
this.IsRewindable = isRewindable;
this.myExtension = null;
this.myGrainReference = null;
this.bindExtLock = new AsyncLock();
}
internal PersistentStreamPullingAgent(
GrainId id,
string strProviderName,
IStreamProviderRuntime runtime,
IStreamPubSub streamPubSub,
QueueId queueId,
PersistentStreamProviderConfig config)
: base(id, runtime.ExecutingSiloAddress, true)
{
if (runtime == null) throw new ArgumentNullException("runtime", "PersistentStreamPullingAgent: runtime reference should not be null");
if (strProviderName == null) throw new ArgumentNullException("runtime", "PersistentStreamPullingAgent: strProviderName should not be null");
QueueId = queueId;
streamProviderName = strProviderName;
providerRuntime = runtime;
pubSub = streamPubSub;
pubSubCache = new Dictionary<StreamId, StreamConsumerCollection>();
safeRandom = new SafeRandom();
this.config = config;
numMessages = 0;
logger = providerRuntime.GetLogger(GrainId + "-" + streamProviderName);
logger.Info((int)ErrorCode.PersistentStreamPullingAgent_01,
"Created {0} {1} for Stream Provider {2} on silo {3} for Queue {4}.",
GetType().Name, GrainId.ToDetailedString(), streamProviderName, Silo, QueueId.ToStringWithHashCode());
string statUniquePostfix = strProviderName + "." + QueueId;
numReadMessagesCounter = CounterStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_READ_MESSAGES, statUniquePostfix));
numSentMessagesCounter = CounterStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_SENT_MESSAGES, statUniquePostfix));
IntValueStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_PUBSUB_CACHE_SIZE, statUniquePostfix), () => pubSubCache.Count);
// TODO: move queue cache size statistics tracking into queue cache implementation once Telemetry APIs and LogStatistics have been reconciled.
//IntValueStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_QUEUE_CACHE_SIZE, statUniquePostfix), () => queueCache != null ? queueCache.Size : 0);
}
internal SimpleMessageStreamProducer(
StreamImpl <T> stream,
string streamProviderName,
IStreamProviderRuntime providerUtilities,
bool fireAndForgetDelivery,
bool optimizeForImmutableData,
IStreamPubSub pubSub,
bool isRewindable,
SerializationManager serializationManager,
ILogger <SimpleMessageStreamProducer <T> > logger)
{
this.stream = stream;
this.streamProviderName = streamProviderName;
providerRuntime = providerUtilities;
this.pubSub = pubSub;
this.serializationManager = serializationManager;
connectedToRendezvous = false;
this.fireAndForgetDelivery = fireAndForgetDelivery;
this.optimizeForImmutableData = optimizeForImmutableData;
IsRewindable = isRewindable;
isDisposed = false;
initLock = new AsyncLock();
this.logger = logger;
ConnectToRendezvous().Ignore();
}
internal SimpleMessageStreamProducerExtension(IStreamProviderRuntime providerRt, bool fireAndForget)
{
providerRuntime = providerRt;
fireAndForgetDelivery = fireAndForget;
remoteConsumers = new Dictionary <StreamId, StreamConsumerExtensionCollection>();
logger = providerRuntime.GetLogger(GetType().Name);
}
public Task Init(string name, IProviderRuntime providerUtilitiesManager, IProviderConfiguration config)
{
if (!stateManager.PresetState(ProviderState.Initialized))
{
return(TaskDone.Done);
}
this.Name = name;
providerRuntime = (IStreamProviderRuntime)providerUtilitiesManager;
this.runtimeClient = this.providerRuntime.ServiceProvider.GetRequiredService <IRuntimeClient>();
fireAndForgetDelivery = config.GetBoolProperty(FIRE_AND_FORGET_DELIVERY, DEFAULT_VALUE_FIRE_AND_FORGET_DELIVERY);
optimizeForImmutableData = config.GetBoolProperty(OPTIMIZE_FOR_IMMUTABLE_DATA, DEFAULT_VALUE_OPTIMIZE_FOR_IMMUTABLE_DATA);
string pubSubTypeString;
pubSubType = !config.Properties.TryGetValue(STREAM_PUBSUB_TYPE, out pubSubTypeString)
? DEFAULT_STREAM_PUBSUB_TYPE
: (StreamPubSubType)Enum.Parse(typeof(StreamPubSubType), pubSubTypeString);
if (pubSubType == StreamPubSubType.ExplicitGrainBasedAndImplicit ||
pubSubType == StreamPubSubType.ExplicitGrainBasedOnly)
{
this.streamSubscriptionManager = this.providerRuntime.ServiceProvider
.GetService <IStreamSubscriptionManagerAdmin>().GetStreamSubscriptionManager(StreamSubscriptionManagerType.ExplicitSubscribeOnly);
}
logger = providerRuntime.GetLogger(this.GetType().Name);
logger.Info("Initialized SimpleMessageStreamProvider with name {0} and with property FireAndForgetDelivery: {1}, OptimizeForImmutableData: {2} " +
"and PubSubType: {3}", Name, fireAndForgetDelivery, optimizeForImmutableData, pubSubType);
stateManager.CommitState();
return(TaskDone.Done);
}
/// <summary>
/// Create stream queue balancer by type requested
/// </summary>
/// <param name="balancerType">queue balancer type to create</param>
/// <param name="strProviderName">name of requesting stream provider</param>
/// <param name="siloStatusOracle">membership services interface.</param>
/// <param name="clusterConfiguration">cluster configuration</param>
/// <param name="runtime">stream provider runtime environment to run in</param>
/// <param name="queueMapper">queue mapper of requesting stream provider</param>
/// <param name="siloMaturityPeriod">Maturity Period of a silo for queue rebalancing purposes</param>
/// <returns>Constructed stream queue balancer</returns>
public static IStreamQueueBalancer Create(
StreamQueueBalancerType balancerType,
string strProviderName,
ISiloStatusOracle siloStatusOracle,
ClusterConfiguration clusterConfiguration,
IStreamProviderRuntime runtime,
IStreamQueueMapper queueMapper,
TimeSpan siloMaturityPeriod)
{
if (string.IsNullOrWhiteSpace(strProviderName))
{
throw new ArgumentNullException("strProviderName");
}
if (siloStatusOracle == null)
{
throw new ArgumentNullException("siloStatusOracle");
}
if (clusterConfiguration == null)
{
throw new ArgumentNullException("clusterConfiguration");
}
if (runtime == null)
{
throw new ArgumentNullException("runtime");
}
if (queueMapper == null)
{
throw new ArgumentNullException("queueMapper");
}
bool isFixed;
switch (balancerType)
{
case StreamQueueBalancerType.ConsistentRingBalancer:
{
// Consider: for now re-use the same ConsistentRingProvider with 1 equally devided range. Remove later.
IConsistentRingProviderForGrains ringProvider = runtime.GetConsistentRingProvider(0, 1);
return new ConsistentRingQueueBalancer(ringProvider, queueMapper);
}
case StreamQueueBalancerType.DynamicAzureDeploymentBalancer:
case StreamQueueBalancerType.StaticAzureDeploymentBalancer:
{
Logger logger = LogManager.GetLogger(typeof(StreamQueueBalancerFactory).Name, LoggerType.Runtime);
var wrapper = AssemblyLoader.LoadAndCreateInstance<IDeploymentConfiguration>(Constants.ORLEANS_AZURE_UTILS_DLL, logger);
isFixed = balancerType == StreamQueueBalancerType.StaticAzureDeploymentBalancer;
return new DeploymentBasedQueueBalancer(siloStatusOracle, wrapper, queueMapper, siloMaturityPeriod, isFixed);
}
case StreamQueueBalancerType.DynamicClusterConfigDeploymentBalancer:
case StreamQueueBalancerType.StaticClusterConfigDeploymentBalancer:
{
IDeploymentConfiguration deploymentConfiguration = new StaticClusterDeploymentConfiguration(clusterConfiguration);
isFixed = balancerType == StreamQueueBalancerType.StaticClusterConfigDeploymentBalancer;
return new DeploymentBasedQueueBalancer(siloStatusOracle, deploymentConfiguration, queueMapper, siloMaturityPeriod, isFixed);
}
default:
{
string error = string.Format("Unsupported balancerType for stream provider. BalancerType: {0}, StreamProvider: {1}", balancerType, strProviderName);
throw new ArgumentOutOfRangeException("balancerType", error);
}
}
}
internal PersistentStreamPullingAgent(
GrainId id,
string strProviderName,
IStreamProviderRuntime runtime,
QueueId queueId,
TimeSpan queueGetPeriod,
TimeSpan initQueueTimeout)
: base(id, runtime.ExecutingSiloAddress, true)
{
if (runtime == null)
{
throw new ArgumentNullException("runtime", "PersistentStreamPullingAgent: runtime reference should not be null");
}
QueueId = queueId;
streamProviderName = strProviderName;
providerRuntime = runtime;
pubSub = runtime.PubSub(StreamPubSubType.GrainBased);
pubSubCache = new Dictionary <StreamId, StreamConsumerCollection>();
safeRandom = new SafeRandom();
this.queueGetPeriod = queueGetPeriod;
this.initQueueTimeout = initQueueTimeout;
numMessages = 0;
logger = providerRuntime.GetLogger(this.GrainId.ToString() + "-" + streamProviderName);
logger.Info((int)ErrorCode.PersistentStreamPullingAgent_01,
"Created {0} {1} for Stream Provider {2} on silo {3} for Queue {4}.",
this.GetType().Name, this.GrainId.ToDetailedString(), streamProviderName, base.Silo, QueueId.ToStringWithHashCode());
numReadMessagesCounter = CounterStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_READ_MESSAGES, strProviderName));
numSentMessagesCounter = CounterStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_SENT_MESSAGES, strProviderName));
}
public async Task Init(string name, IProviderRuntime providerUtilitiesManager, IProviderConfiguration config)
{
if (String.IsNullOrEmpty(name))
{
throw new ArgumentNullException("name");
}
if (providerUtilitiesManager == null)
{
throw new ArgumentNullException("providerUtilitiesManager");
}
if (config == null)
{
throw new ArgumentNullException("config");
}
Name = name;
providerRuntime = (IStreamProviderRuntime)providerUtilitiesManager;
logger = providerRuntime.GetLogger(this.GetType().Name);
IQueueAdapterFactory factory = new TAdapterFactory();
factory.Init(config, Name, logger);
queueAdapter = await factory.CreateAdapter();
string timePeriod;
if (!config.Properties.TryGetValue(GET_QUEUE_MESSAGES_TIMER_PERIOD, out timePeriod))
{
getQueueMsgsTimerPeriod = DEFAULT_GET_QUEUE_MESSAGES_TIMER_PERIOD;
}
else
{
getQueueMsgsTimerPeriod = ConfigUtilities.ParseTimeSpan(timePeriod,
"Invalid time value for the " + GET_QUEUE_MESSAGES_TIMER_PERIOD + " property in the provider config values.");
}
string timeout;
if (!config.Properties.TryGetValue(INIT_QUEUE_TIMEOUT, out timeout))
{
initQueueTimeout = DEFAULT_INIT_QUEUE_TIMEOUT;
}
else
{
initQueueTimeout = ConfigUtilities.ParseTimeSpan(timeout,
"Invalid time value for the " + INIT_QUEUE_TIMEOUT + " property in the provider config values.");
}
string balanceTypeString;
balancerType = !config.Properties.TryGetValue(QUEUE_BALANCER_TYPE, out balanceTypeString)
? DEFAULT_STREAM_QUEUE_BALANCER_TYPE
: (StreamQueueBalancerType)Enum.Parse(typeof(StreamQueueBalancerType), balanceTypeString);
logger.Info("Initialized PersistentStreamProvider<{0}> with name {1}, {2} = {3}, {4} = {5} and with Adapter {6}.",
typeof(TAdapterFactory).Name, Name,
GET_QUEUE_MESSAGES_TIMER_PERIOD, getQueueMsgsTimerPeriod,
INIT_QUEUE_TIMEOUT, this.initQueueTimeout,
queueAdapter.Name);
}
public StreamProviderManagerAgent(Silo silo, IStreamProviderRuntime streamProviderRuntime)
: base(Constants.StreamProviderManagerAgentSystemTargetId, silo.SiloAddress)
{
logger = LogManager.GetLogger("StreamProviderUpdateAgent", LoggerType.Runtime);
this.streamProviderManager = (StreamProviderManager)silo.StreamProviderManager;
providerConfigurations = silo.GlobalConfig.ProviderConfigurations;
this.streamProviderRuntime = streamProviderRuntime;
nonReentrancyGuarantor = new AsyncSerialExecutor();
}
internal SimpleMessageStreamProducerExtension(IStreamProviderRuntime providerRt, IStreamPubSub pubsub, bool fireAndForget, bool optimizeForImmutable)
{
providerRuntime = providerRt;
streamPubSub = pubsub;
fireAndForgetDelivery = fireAndForget;
optimizeForImmutableData = optimizeForImmutable;
remoteConsumers = new Dictionary<StreamId, StreamConsumerExtensionCollection>();
logger = providerRuntime.GetLogger(GetType().Name);
}
/// <summary>
/// Create stream queue balancer by type requested
/// </summary>
/// <param name="balancerType">queue balancer type to create</param>
/// <param name="strProviderName">name of requesting stream provider</param>
/// <param name="siloStatusOracle">membership services interface.</param>
/// <param name="clusterConfiguration">cluster configuration</param>
/// <param name="runtime">stream provider runtime environment to run in</param>
/// <param name="queueMapper">queue mapper of requesting stream provider</param>
/// <returns>Constructed stream queue balancer</returns>
public static IStreamQueueBalancer Create(
StreamQueueBalancerType balancerType,
string strProviderName,
ISiloStatusOracle siloStatusOracle,
ClusterConfiguration clusterConfiguration,
IStreamProviderRuntime runtime,
IStreamQueueMapper queueMapper)
{
if (string.IsNullOrWhiteSpace(strProviderName))
{
throw new ArgumentNullException("strProviderName");
}
if (siloStatusOracle == null)
{
throw new ArgumentNullException("siloStatusOracle");
}
if (clusterConfiguration == null)
{
throw new ArgumentNullException("clusterConfiguration");
}
if (runtime == null)
{
throw new ArgumentNullException("runtime");
}
if (queueMapper == null)
{
throw new ArgumentNullException("queueMapper");
}
switch (balancerType)
{
case StreamQueueBalancerType.ConsistentRingBalancer:
{
// Consider: for now re-use the same ConsistentRingProvider with 1 equally devided range. Remove later.
IConsistentRingProviderForGrains ringProvider = runtime.GetConsistentRingProvider(0, 1);
return(new ConsistentRingQueueBalancer(ringProvider, queueMapper));
}
case StreamQueueBalancerType.AzureDeploymentBalancer:
{
TraceLogger logger = TraceLogger.GetLogger(typeof(StreamQueueBalancerFactory).Name, TraceLogger.LoggerType.Runtime);
var wrapper = AssemblyLoader.LoadAndCreateInstance <IDeploymentConfiguration>(Constants.ORLEANS_AZURE_UTILS_DLL, logger);
return(new DeploymentBasedQueueBalancer(siloStatusOracle, wrapper, queueMapper));
}
case StreamQueueBalancerType.StaticClusterDeploymentBalancer:
{
IDeploymentConfiguration deploymentConfiguration = new StaticClusterDeploymentConfiguration(clusterConfiguration);
return(new DeploymentBasedQueueBalancer(siloStatusOracle, deploymentConfiguration, queueMapper));
}
default:
{
string error = string.Format("Unsupported balancerType for stream provider. BalancerType: {0}, StreamProvider: {1}", balancerType, strProviderName);
throw new ArgumentOutOfRangeException("balancerType", error);
}
}
}
internal SimpleMessageStreamProducerExtension(IStreamProviderRuntime providerRt, IStreamPubSub pubsub, bool fireAndForget, bool optimizeForImmutable)
{
providerRuntime = providerRt;
streamPubSub = pubsub;
fireAndForgetDelivery = fireAndForget;
optimizeForImmutableData = optimizeForImmutable;
remoteConsumers = new Dictionary <StreamId, StreamConsumerExtensionCollection>();
logger = providerRuntime.GetLogger(GetType().Name);
}
public StreamProviderManagerAgent(Silo silo, List<IProvider> allSiloProviders, IStreamProviderRuntime streamProviderRuntime)
: base(Constants.StreamProviderManagerAgentSystemTargetId, silo.SiloAddress)
{
logger = LogManager.GetLogger("StreamProviderUpdateAgent", LoggerType.Runtime);
this.streamProviderManager = (StreamProviderManager)silo.StreamProviderManager;
providerConfigurations = silo.GlobalConfig.ProviderConfigurations;
this.allSiloProviders = allSiloProviders;
this.streamProviderRuntime = streamProviderRuntime;
nonReentrancyGuarantor = new AsyncSerialExecutor();
}
public StreamProviderManagerAgent(Silo silo, IStreamProviderRuntime streamProviderRuntime, ILoggerFactory loggerFactory)
: base(Constants.StreamProviderManagerAgentSystemTargetId, silo.SiloAddress, loggerFactory)
{
logger = loggerFactory.CreateLogger <StreamProviderManagerAgent>();
this.streamProviderManager = (StreamProviderManager)silo.StreamProviderManager;
providerConfigurations = silo.GlobalConfig.ProviderConfigurations;
this.streamProviderRuntime = streamProviderRuntime;
nonReentrancyGuarantor = new AsyncSerialExecutor();
}
internal SimpleMessageStreamProducerExtension(IStreamProviderRuntime providerRt, IStreamPubSub pubsub, ILoggerFactory loggerFactory, bool fireAndForget, bool optimizeForImmutable)
{
providerRuntime = providerRt;
streamPubSub = pubsub;
fireAndForgetDelivery = fireAndForget;
optimizeForImmutableData = optimizeForImmutable;
remoteConsumers = new Dictionary <StreamId, StreamConsumerExtensionCollection>();
logger = loggerFactory.CreateLogger <SimpleMessageStreamProducerExtension>();
this.loggerFactory = loggerFactory;
}
internal async Task LoadStreamProviders(
IDictionary<string, ProviderCategoryConfiguration> configs,
IStreamProviderRuntime providerRuntime)
{
appStreamProviders = new ProviderLoader<IStreamProviderImpl>();
if (!configs.ContainsKey(ProviderCategoryConfiguration.STREAM_PROVIDER_CATEGORY_NAME)) return;
appStreamProviders.LoadProviders(configs[ProviderCategoryConfiguration.STREAM_PROVIDER_CATEGORY_NAME].Providers, this);
await appStreamProviders.InitProviders(providerRuntime);
}
public ConsistentRingQueueBalancer(IStreamProviderRuntime streamProviderRuntime)
{
if (streamProviderRuntime == null)
{
throw new ArgumentNullException("streamProviderRuntime");
}
var ringProvider = streamProviderRuntime.GetConsistentRingProvider(0, 1);
myRange = ringProvider.GetMyRange();
ringProvider.SubscribeToRangeChangeEvents(this);
}
internal async Task LoadStreamProviders(
IDictionary <string, ProviderCategoryConfiguration> configs,
IStreamProviderRuntime providerRuntime)
{
if (!configs.ContainsKey(ProviderCategoryConfiguration.STREAM_PROVIDER_CATEGORY_NAME))
{
return;
}
appStreamProviders.LoadProviders(configs[ProviderCategoryConfiguration.STREAM_PROVIDER_CATEGORY_NAME].Providers, this);
await appStreamProviders.InitProviders(providerRuntime);
}
public Task Init(string name, IProviderRuntime providerUtilitiesManager, IProviderConfiguration config)
{
this.Name = name;
providerRuntime = (IStreamProviderRuntime)providerUtilitiesManager;
string fireAndForgetDeliveryStr;
fireAndForgetDelivery = config.Properties.TryGetValue(FIRE_AND_FORGET_DELIVERY, out fireAndForgetDeliveryStr) && Boolean.Parse(fireAndForgetDeliveryStr);
logger = providerRuntime.GetLogger(this.GetType().Name);
logger.Info("Initialized SimpleMessageStreamProvider with name {0} and with property FireAndForgetDelivery: {1}.", Name, fireAndForgetDelivery);
return(TaskDone.Done);
}
internal PersistentStreamProducer(StreamImpl <T> stream, IStreamProviderRuntime providerUtilities, IQueueAdapter queueAdapter, bool isRewindable)
{
this.stream = stream;
this.queueAdapter = queueAdapter;
IsRewindable = isRewindable;
var logger = providerUtilities.GetLogger(this.GetType().Name);
if (logger.IsVerbose)
{
logger.Verbose("Created PersistentStreamProducer for stream {0}, of type {1}, and with Adapter: {2}.",
stream.ToString(), typeof(T), this.queueAdapter.Name);
}
}
internal PersistentStreamProducer(StreamImpl <T> stream, IStreamProviderRuntime providerUtilities, IQueueAdapter queueAdapter, bool isRewindable, DeepCopier deepCopier)
{
this.stream = stream;
this.queueAdapter = queueAdapter;
this.deepCopier = deepCopier;
IsRewindable = isRewindable;
var logger = providerUtilities.ServiceProvider.GetRequiredService <ILogger <PersistentStreamProducer <T> > >();
if (logger.IsEnabled(LogLevel.Debug))
{
logger.LogDebug("Created PersistentStreamProducer for stream {StreamId}, of type {ElementType}, and with Adapter: {QueueAdapterName}.",
stream.ToString(), typeof(T), this.queueAdapter.Name);
}
}
internal async Task LoadStreamProviders(
IDictionary <string, ProviderCategoryConfiguration> configs,
IStreamProviderRuntime providerRuntime)
{
appStreamProviders = new ProviderLoader <IStreamProvider>();
if (!configs.ContainsKey("Stream"))
{
return;
}
appStreamProviders.LoadProviders(configs["Stream"].Providers, this);
await appStreamProviders.InitProviders(providerRuntime);
}
internal PersistentStreamProducer(StreamImpl <T> stream, IStreamProviderRuntime providerUtilities, IQueueAdapter queueAdapter, bool isRewindable, SerializationManager serializationManager)
{
this.stream = stream;
this.queueAdapter = queueAdapter;
this.serializationManager = serializationManager;
IsRewindable = isRewindable;
var logger = providerUtilities.ServiceProvider.GetRequiredService <ILoggerFactory>().CreateLogger(this.GetType().Name);
if (logger.IsEnabled(LogLevel.Debug))
{
logger.Debug("Created PersistentStreamProducer for stream {0}, of type {1}, and with Adapter: {2}.",
stream.ToString(), typeof(T), this.queueAdapter.Name);
}
}
public async Task Init(string name, IProviderRuntime providerUtilitiesManager, IProviderConfiguration config)
{
if (!stateManager.PresetState(ProviderState.Initialized))
{
return;
}
if (String.IsNullOrEmpty(name))
{
throw new ArgumentNullException("name");
}
if (providerUtilitiesManager == null)
{
throw new ArgumentNullException("providerUtilitiesManager");
}
if (config == null)
{
throw new ArgumentNullException("config");
}
Name = name;
providerRuntime = (IStreamProviderRuntime)providerUtilitiesManager;
logger = providerRuntime.GetLogger(this.GetType().Name);
adapterFactory = new TAdapterFactory();
adapterFactory.Init(config, Name, logger, providerRuntime.ServiceProvider);
queueAdapter = await adapterFactory.CreateAdapter();
myConfig = new PersistentStreamProviderConfig(config);
string startup;
if (config.Properties.TryGetValue(StartupStatePropertyName, out startup))
{
if (!Enum.TryParse(startup, true, out startupState))
{
throw new ArgumentException(
String.Format("Unsupported value '{0}' for configuration parameter {1} of stream provider {2}.", startup, StartupStatePropertyName, config.Name));
}
}
else
{
startupState = StartupStateDefaultValue;
}
logger.Info("Initialized PersistentStreamProvider<{0}> with name {1}, Adapter {2} and config {3}, {4} = {5}.",
typeof(TAdapterFactory).Name,
Name,
queueAdapter.Name,
myConfig,
StartupStatePropertyName, startupState);
stateManager.CommitState();
}
internal PersistentStreamPullingManager(
GrainId id,
string strProviderName,
IStreamProviderRuntime runtime,
IStreamPubSub streamPubSub,
IQueueAdapterFactory adapterFactory,
IStreamQueueBalancer streamQueueBalancer,
TimeSpan queueGetPeriod,
TimeSpan initQueueTimeout,
TimeSpan maxEvenDeliveryTime)
: base(id, runtime.ExecutingSiloAddress)
{
if (string.IsNullOrWhiteSpace(strProviderName))
{
throw new ArgumentNullException("strProviderName");
}
if (runtime == null)
{
throw new ArgumentNullException("runtime", "IStreamProviderRuntime runtime reference should not be null");
}
if (streamPubSub == null)
{
throw new ArgumentNullException("streamPubSub", "StreamPubSub reference should not be null");
}
if (streamQueueBalancer == null)
{
throw new ArgumentNullException("streamQueueBalancer", "IStreamQueueBalancer streamQueueBalancer reference should not be null");
}
queuesToAgentsMap = new Dictionary<QueueId, PersistentStreamPullingAgent>();
streamProviderName = strProviderName;
providerRuntime = runtime;
pubSub = streamPubSub;
this.queueGetPeriod = queueGetPeriod;
this.initQueueTimeout = initQueueTimeout;
this.maxEvenDeliveryTime = maxEvenDeliveryTime;
nonReentrancyGuarantor = new AsyncSerialExecutor();
latestRingNotificationSequenceNumber = 0;
latestCommandNumber = 0;
queueBalancer = streamQueueBalancer;
this.adapterFactory = adapterFactory;
queueAdapterCache = adapterFactory.GetQueueAdapterCache();
logger = providerRuntime.GetLogger(GetType().Name + "-" + streamProviderName);
logger.Info((int)ErrorCode.PersistentStreamPullingManager_01, "Created {0} for Stream Provider {1}.", GetType().Name, streamProviderName);
IntValueStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_PULLING_AGENTS, strProviderName), () => queuesToAgentsMap.Count);
}
internal PersistentStreamPullingManager(
GrainId id,
string strProviderName,
IStreamProviderRuntime runtime,
IStreamPubSub streamPubSub,
IQueueAdapterFactory adapterFactory,
IStreamQueueBalancer streamQueueBalancer,
PersistentStreamProviderConfig config)
: base(id, runtime.ExecutingSiloAddress)
{
if (string.IsNullOrWhiteSpace(strProviderName))
{
throw new ArgumentNullException("strProviderName");
}
if (runtime == null)
{
throw new ArgumentNullException("runtime", "IStreamProviderRuntime runtime reference should not be null");
}
if (streamPubSub == null)
{
throw new ArgumentNullException("streamPubSub", "StreamPubSub reference should not be null");
}
if (streamQueueBalancer == null)
{
throw new ArgumentNullException("streamQueueBalancer", "IStreamQueueBalancer streamQueueBalancer reference should not be null");
}
queuesToAgentsMap = new Dictionary<QueueId, PersistentStreamPullingAgent>();
streamProviderName = strProviderName;
providerRuntime = runtime;
pubSub = streamPubSub;
this.config = config;
nonReentrancyGuarantor = new AsyncSerialExecutor();
latestRingNotificationSequenceNumber = 0;
latestCommandNumber = 0;
queueBalancer = streamQueueBalancer;
this.adapterFactory = adapterFactory;
queueAdapterCache = adapterFactory.GetQueueAdapterCache();
logger = TraceLogger.GetLogger(GetType().Name + "-" + streamProviderName, TraceLogger.LoggerType.Provider);
Log(ErrorCode.PersistentStreamPullingManager_01, "Created {0} for Stream Provider {1}.", GetType().Name, streamProviderName);
IntValueStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_PULLING_AGENTS, strProviderName), () => queuesToAgentsMap.Count);
queuePrintTimer = base.RegisterTimer(AsyncTimerCallback, null, QUEUES_PRINT_PERIOD, QUEUES_PRINT_PERIOD);
}
public StreamLifecycleConsumerInternalGrain(ISiloRuntimeClient runtimeClient, IStreamProviderRuntime streamProviderRuntime)
: base(runtimeClient, streamProviderRuntime)
{
}
