public async Task <IPersistentStreamPullingManager> InitializePullingAgents(
string streamProviderName,
IQueueAdapterFactory adapterFactory,
IQueueAdapter queueAdapter)
{
IStreamQueueBalancer queueBalancer = CreateQueueBalancer(streamProviderName);
var managerId = LegacyGrainId.NewSystemTargetGrainIdByTypeCode(Constants.PULLING_AGENTS_MANAGER_SYSTEM_TARGET_TYPE_CODE);
var pubsubOptions = this.ServiceProvider.GetOptionsByName <StreamPubSubOptions>(streamProviderName);
var pullingAgentOptions = this.ServiceProvider.GetOptionsByName <StreamPullingAgentOptions>(streamProviderName);
var manager = new PersistentStreamPullingManager(
managerId,
streamProviderName,
this,
this.PubSub(pubsubOptions.PubSubType),
adapterFactory,
queueBalancer,
pullingAgentOptions,
this.loggerFactory,
this.siloDetails.SiloAddress);
this.RegisterSystemTarget(manager);
// Init the manager only after it was registered locally.
var pullingAgentManager = manager.AsReference <IPersistentStreamPullingManager>();
// Need to call it as a grain reference though.
await pullingAgentManager.Initialize(queueAdapter.AsImmutable());
return(pullingAgentManager);
}
public Publisher(ILogController logController, IOptions <ConnectionStrings> connectionStrings, IOptions <AzureServiceBusConfiguration> serviceBusConfiguration) : base(logController, connectionStrings, serviceBusConfiguration)
{
_queueManager = this.BuildDefaultQueueAdapter();
logger = this.BuildLogger();
_queueManager.Initialize();
this.logController = logController;
}
public async Task Init(string name, IProviderRuntime providerUtilitiesManager, IProviderConfiguration config)
{
if (!stateManager.PresetState(ProviderState.Initialized))
{
return;
}
if (String.IsNullOrEmpty(name))
{
throw new ArgumentNullException("name");
}
if (providerUtilitiesManager == null)
{
throw new ArgumentNullException("providerUtilitiesManager");
}
if (config == null)
{
throw new ArgumentNullException("config");
}
Name = name;
providerRuntime = (IStreamProviderRuntime)providerUtilitiesManager;
logger = providerRuntime.GetLogger(this.GetType().Name);
adapterFactory = new TAdapterFactory();
// Temporary change, but we need GrainFactory inside ServiceProvider for now,
// so will change it back as soon as we have an action item to add GrainFactory to ServiceProvider.
adapterFactory.Init(config, Name, logger, new GrainFactoryServiceProvider(providerRuntime));
queueAdapter = await adapterFactory.CreateAdapter();
myConfig = new PersistentStreamProviderConfig(config);
this.providerConfig = config;
this.serializationManager = this.providerRuntime.ServiceProvider.GetRequiredService <SerializationManager>();
this.runtimeClient = this.providerRuntime.ServiceProvider.GetRequiredService <IRuntimeClient>();
if (this.myConfig.PubSubType == StreamPubSubType.ExplicitGrainBasedAndImplicit ||
this.myConfig.PubSubType == StreamPubSubType.ExplicitGrainBasedOnly)
{
this.streamSubscriptionManager = this.providerRuntime.ServiceProvider
.GetService <IStreamSubscriptionManagerAdmin>().GetStreamSubscriptionManager(StreamSubscriptionManagerType.ExplicitSubscribeOnly);
}
string startup;
if (config.Properties.TryGetValue(StartupStatePropertyName, out startup))
{
if (!Enum.TryParse(startup, true, out startupState))
{
throw new ArgumentException(
String.Format("Unsupported value '{0}' for configuration parameter {1} of stream provider {2}.", startup, StartupStatePropertyName, config.Name));
}
}
else
{
startupState = StartupStateDefaultValue;
}
logger.Info("Initialized PersistentStreamProvider<{0}> with name {1}, Adapter {2} and config {3}, {4} = {5}.",
typeof(TAdapterFactory).Name,
Name,
queueAdapter.Name,
myConfig,
StartupStatePropertyName, startupState);
stateManager.CommitState();
}
public async Task <IPersistentStreamPullingManager> InitializePullingAgents(
string streamProviderName,
IQueueAdapterFactory adapterFactory,
IQueueAdapter queueAdapter)
{
IStreamQueueBalancer queueBalancer = CreateQueueBalancer(streamProviderName);
var managerId = SystemTargetGrainId.Create(Constants.StreamPullingAgentManagerType, this.siloDetails.SiloAddress, streamProviderName);
var pubsubOptions = this.ServiceProvider.GetOptionsByName <StreamPubSubOptions>(streamProviderName);
var pullingAgentOptions = this.ServiceProvider.GetOptionsByName <StreamPullingAgentOptions>(streamProviderName);
var filter = this.ServiceProvider.GetServiceByName <IStreamFilter>(streamProviderName) ?? new NoOpStreamFilter();
var manager = new PersistentStreamPullingManager(
managerId,
streamProviderName,
this.PubSub(pubsubOptions.PubSubType),
adapterFactory,
queueBalancer,
filter,
pullingAgentOptions,
this.loggerFactory,
this.siloDetails.SiloAddress,
queueAdapter);
var catalog = this.ServiceProvider.GetRequiredService <Catalog>();
catalog.RegisterSystemTarget(manager);
// Init the manager only after it was registered locally.
var pullingAgentManager = manager.AsReference <IPersistentStreamPullingManager>();
// Need to call it as a grain reference though.
await pullingAgentManager.Initialize();
return(pullingAgentManager);
}
/// <summary>
/// Initializes a new instance of the <see cref="Publisher<T>"/> class.
/// </summary>
/// <param name="queueManager">The queue manager.</param>
/// <param name="logger">The logger.</param>
public Publisher(IQueueAdapter queueManager, IBusLogger logger, ILogController logController, IOptions <ConnectionStrings> connectionStrings, IOptions <AzureServiceBusConfiguration> serviceBusConfiguration) : base(logController, connectionStrings, serviceBusConfiguration)
{
this.logger = logger;
this._queueManager = queueManager;
queueManager.Initialize();
this.logController = logController;
}
/// <summary>
/// Take responsibility for a new queues that was assigned to me via a new range.
/// We first store the new queue in our internal data structure, try to initialize it and start a pumping timer.
/// ERROR HANDLING:
/// The resposibility to handle initializatoion and shutdown failures is inside the INewQueueAdapterReceiver code.
/// The agent will call Initialize once and log an error. It will not call initiliaze again.
/// The receiver itself may attempt later to recover from this error and do initialization again.
/// The agent will assume initialization has succeeded and will subsequently start calling pumping receive.
/// Same applies to shutdown.
/// </summary>
/// <param name="qAdapter"></param>
/// <param name="queueAdapterCache"></param>
/// <param name="failureHandler"></param>
/// <returns></returns>
public async Task Initialize(Immutable <IQueueAdapter> qAdapter, Immutable <IQueueAdapterCache> queueAdapterCache, Immutable <IStreamFailureHandler> failureHandler)
{
if (qAdapter.Value == null)
{
throw new ArgumentNullException("qAdapter", "Init: queueAdapter should not be null");
}
if (failureHandler.Value == null)
{
throw new ArgumentNullException("failureHandler", "Init: streamDeliveryFailureHandler should not be null");
}
logger.Info((int)ErrorCode.PersistentStreamPullingAgent_02, "Init of {0} {1} on silo {2} for queue {3}.",
GetType().Name, GrainId.ToDetailedString(), Silo, QueueId.ToStringWithHashCode());
// Remove cast once we cleanup
queueAdapter = qAdapter.Value;
streamFailureHandler = failureHandler.Value;
try
{
receiver = queueAdapter.CreateReceiver(QueueId);
}
catch (Exception exc)
{
logger.Error((int)ErrorCode.PersistentStreamPullingAgent_02, String.Format("Exception while calling IQueueAdapter.CreateNewReceiver."), exc);
return;
}
try
{
if (queueAdapterCache.Value != null)
{
queueCache = queueAdapterCache.Value.CreateQueueCache(QueueId);
}
}
catch (Exception exc)
{
logger.Error((int)ErrorCode.PersistentStreamPullingAgent_23, String.Format("Exception while calling IQueueAdapterCache.CreateQueueCache."), exc);
return;
}
try
{
var task = OrleansTaskExtentions.SafeExecute(() => receiver.Initialize(initQueueTimeout));
task = task.LogException(logger, ErrorCode.PersistentStreamPullingAgent_03, String.Format("QueueAdapterReceiver {0} failed to Initialize.", QueueId.ToStringWithHashCode()));
await task;
}
catch
{
// Just ignore this exception and proceed as if Initialize has succeeded.
// We already logged individual exceptions for individual calls to Initialize. No need to log again.
}
// Setup a reader for a new receiver.
// Even if the receiver failed to initialise, treat it as OK and start pumping it. It's receiver responsibility to retry initialization.
var randomTimerOffset = safeRandom.NextTimeSpan(queueGetPeriod);
timer = providerRuntime.RegisterTimer(AsyncTimerCallback, QueueId, randomTimerOffset, queueGetPeriod);
logger.Info((int)ErrorCode.PersistentStreamPullingAgent_04, "Taking queue {0} under my responsibility.", QueueId.ToStringWithHashCode());
}
public async Task Init(string name, IProviderRuntime providerUtilitiesManager, IProviderConfiguration config)
{
if (String.IsNullOrEmpty(name))
{
throw new ArgumentNullException("name");
}
if (providerUtilitiesManager == null)
{
throw new ArgumentNullException("providerUtilitiesManager");
}
if (config == null)
{
throw new ArgumentNullException("config");
}
Name = name;
providerRuntime = (IStreamProviderRuntime)providerUtilitiesManager;
logger = providerRuntime.GetLogger(this.GetType().Name);
IQueueAdapterFactory factory = new TAdapterFactory();
factory.Init(config, Name, logger);
queueAdapter = await factory.CreateAdapter();
string timePeriod;
if (!config.Properties.TryGetValue(GET_QUEUE_MESSAGES_TIMER_PERIOD, out timePeriod))
{
getQueueMsgsTimerPeriod = DEFAULT_GET_QUEUE_MESSAGES_TIMER_PERIOD;
}
else
{
getQueueMsgsTimerPeriod = ConfigUtilities.ParseTimeSpan(timePeriod,
"Invalid time value for the " + GET_QUEUE_MESSAGES_TIMER_PERIOD + " property in the provider config values.");
}
string timeout;
if (!config.Properties.TryGetValue(INIT_QUEUE_TIMEOUT, out timeout))
{
initQueueTimeout = DEFAULT_INIT_QUEUE_TIMEOUT;
}
else
{
initQueueTimeout = ConfigUtilities.ParseTimeSpan(timeout,
"Invalid time value for the " + INIT_QUEUE_TIMEOUT + " property in the provider config values.");
}
string balanceTypeString;
balancerType = !config.Properties.TryGetValue(QUEUE_BALANCER_TYPE, out balanceTypeString)
? DEFAULT_STREAM_QUEUE_BALANCER_TYPE
: (StreamQueueBalancerType)Enum.Parse(typeof(StreamQueueBalancerType), balanceTypeString);
logger.Info("Initialized PersistentStreamProvider<{0}> with name {1}, {2} = {3}, {4} = {5} and with Adapter {6}.",
typeof(TAdapterFactory).Name, Name,
GET_QUEUE_MESSAGES_TIMER_PERIOD, getQueueMsgsTimerPeriod,
INIT_QUEUE_TIMEOUT, this.initQueueTimeout,
queueAdapter.Name);
}
/// <summary>
/// Builds the publisher.
/// </summary>
/// <returns></returns>
public Publisher <T> BuildPublisher()
{
IQueueAdapter queueAdapter = BuildDefaultQueueAdapter();
IBusLogger logger = BuildLogger();
var publisher = new Publisher <T>(queueAdapter, logger, this.logController, this.connectionStrings, this.serviceBusConfiguration);
return(publisher);
}
/// <summary>
/// Sends the message.
/// </summary>
/// <param name="message">The message.</param>
public virtual void SendMessage(T message, string azureServiceBusTopicName)
{
_queueManager = BuildAzureQueueAdapterForTopic(azureServiceBusTopicName);
var envelope = new MessageEnvelope(message);
SendEnvelope(envelope);
}
public Task StartPullingAgents(
string streamProviderName,
StreamQueueBalancerType balancerType,
IQueueAdapter queueAdapter,
TimeSpan getQueueMsgsTimerPeriod,
TimeSpan initQueueTimeout)
{
return(TaskDone.Done);
}
private void InitializeInternal(IQueueAdapter qAdapter, IQueueAdapterCache queueAdapterCache, IStreamFailureHandler failureHandler)
{
logger.Info(ErrorCode.PersistentStreamPullingAgent_02, "Init of {0} {1} on silo {2} for queue {3}.",
GetType().Name, ((ISystemTargetBase)this).GrainId.ToDetailedString(), Silo, QueueId.ToStringWithHashCode());
// Remove cast once we cleanup
queueAdapter = qAdapter;
streamFailureHandler = failureHandler;
lastTimeCleanedPubSubCache = DateTime.UtcNow;
try
{
receiver = queueAdapter.CreateReceiver(QueueId);
}
catch (Exception exc)
{
logger.Error(ErrorCode.PersistentStreamPullingAgent_02, "Exception while calling IQueueAdapter.CreateNewReceiver.", exc);
throw;
}
try
{
if (queueAdapterCache != null)
{
queueCache = queueAdapterCache.CreateQueueCache(QueueId);
}
}
catch (Exception exc)
{
logger.Error(ErrorCode.PersistentStreamPullingAgent_23, "Exception while calling IQueueAdapterCache.CreateQueueCache.", exc);
throw;
}
try
{
receiverInitTask = OrleansTaskExtentions.SafeExecute(() => receiver.Initialize(config.InitQueueTimeout))
.LogException(logger, ErrorCode.PersistentStreamPullingAgent_03, $"QueueAdapterReceiver {QueueId.ToStringWithHashCode()} failed to Initialize.");
receiverInitTask.Ignore();
}
catch
{
// Just ignore this exception and proceed as if Initialize has succeeded.
// We already logged individual exceptions for individual calls to Initialize. No need to log again.
}
// Setup a reader for a new receiver.
// Even if the receiver failed to initialise, treat it as OK and start pumping it. It's receiver responsibility to retry initialization.
var randomTimerOffset = safeRandom.NextTimeSpan(config.GetQueueMsgsTimerPeriod);
timer = RegisterTimer(AsyncTimerCallback, QueueId, randomTimerOffset, config.GetQueueMsgsTimerPeriod);
IntValueStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_PUBSUB_CACHE_SIZE, StatisticUniquePostfix), () => pubSubCache.Count);
logger.Info((int)ErrorCode.PersistentStreamPullingAgent_04, "Taking queue {0} under my responsibility.", QueueId.ToStringWithHashCode());
}
public ImagesService(IImagesRepository imagesRepository, IFileStorageAdapter storageAdapter, IQueueAdapter queueAdapter, ICompoundImagesRepository compoundImagesRepository, ICompoundImageMappingsRepository compoundImageMappingsRepository, IMapsAnalyser mapsAnalyser, IMapsRepository mapsRepository, IConfiguration configuration)
{
_imagesRepository = imagesRepository;
_queueAdapter = queueAdapter;
_storageAdapter = storageAdapter;
_compoundImagesRepository = compoundImagesRepository;
_compoundImageMappingsRepository = compoundImageMappingsRepository;
_mapsAnalyser = mapsAnalyser;
_mapsRepository = mapsRepository;
_storageContainerName = configuration.GetValue <string>("Azure:CloudBlobImageContainerName");
}
internal PersistentStreamProducer(StreamImpl <T> stream, IStreamProviderRuntime providerUtilities, IQueueAdapter queueAdapter, bool isRewindable)
{
this.stream = stream;
this.queueAdapter = queueAdapter;
IsRewindable = isRewindable;
var logger = providerUtilities.GetLogger(this.GetType().Name);
if (logger.IsVerbose)
{
logger.Verbose("Created PersistentStreamProducer for stream {0}, of type {1}, and with Adapter: {2}.",
stream.ToString(), typeof(T), this.queueAdapter.Name);
}
}
public async Task Init(string name, IProviderRuntime providerUtilitiesManager, IProviderConfiguration config)
{
if (!stateManager.PresetState(ProviderState.Initialized))
{
return;
}
if (String.IsNullOrEmpty(name))
{
throw new ArgumentNullException("name");
}
if (providerUtilitiesManager == null)
{
throw new ArgumentNullException("providerUtilitiesManager");
}
if (config == null)
{
throw new ArgumentNullException("config");
}
Name = name;
providerRuntime = (IStreamProviderRuntime)providerUtilitiesManager;
logger = providerRuntime.GetLogger(this.GetType().Name);
adapterFactory = new TAdapterFactory();
adapterFactory.Init(config, Name, logger, providerRuntime.ServiceProvider);
queueAdapter = await adapterFactory.CreateAdapter();
myConfig = new PersistentStreamProviderConfig(config);
string startup;
if (config.Properties.TryGetValue(StartupStatePropertyName, out startup))
{
if (!Enum.TryParse(startup, true, out startupState))
{
throw new ArgumentException(
String.Format("Unsupported value '{0}' for configuration parameter {1} of stream provider {2}.", startup, StartupStatePropertyName, config.Name));
}
}
else
{
startupState = StartupStateDefaultValue;
}
logger.Info("Initialized PersistentStreamProvider<{0}> with name {1}, Adapter {2} and config {3}, {4} = {5}.",
typeof(TAdapterFactory).Name,
Name,
queueAdapter.Name,
myConfig,
StartupStatePropertyName, startupState);
stateManager.CommitState();
}
internal PersistentStreamProducer(StreamImpl <T> stream, IStreamProviderRuntime providerUtilities, IQueueAdapter queueAdapter, bool isRewindable, DeepCopier deepCopier)
{
this.stream = stream;
this.queueAdapter = queueAdapter;
this.deepCopier = deepCopier;
IsRewindable = isRewindable;
var logger = providerUtilities.ServiceProvider.GetRequiredService <ILogger <PersistentStreamProducer <T> > >();
if (logger.IsEnabled(LogLevel.Debug))
{
logger.LogDebug("Created PersistentStreamProducer for stream {StreamId}, of type {ElementType}, and with Adapter: {QueueAdapterName}.",
stream.ToString(), typeof(T), this.queueAdapter.Name);
}
}
internal PersistentStreamProducer(StreamImpl <T> stream, IStreamProviderRuntime providerUtilities, IQueueAdapter queueAdapter, bool isRewindable, SerializationManager serializationManager)
{
this.stream = stream;
this.queueAdapter = queueAdapter;
this.serializationManager = serializationManager;
IsRewindable = isRewindable;
var logger = providerUtilities.ServiceProvider.GetRequiredService <ILoggerFactory>().CreateLogger(this.GetType().Name);
if (logger.IsEnabled(LogLevel.Debug))
{
logger.Debug("Created PersistentStreamProducer for stream {0}, of type {1}, and with Adapter: {2}.",
stream.ToString(), typeof(T), this.queueAdapter.Name);
}
}
internal PersistentStreamPullingManager(
SystemTargetGrainId managerId,
string strProviderName,
IStreamPubSub streamPubSub,
IQueueAdapterFactory adapterFactory,
IStreamQueueBalancer streamQueueBalancer,
IStreamFilter streamFilter,
StreamPullingAgentOptions options,
ILoggerFactory loggerFactory,
SiloAddress siloAddress,
IQueueAdapter queueAdapter)
: base(managerId, siloAddress, lowPriority: false, loggerFactory)
{
if (string.IsNullOrWhiteSpace(strProviderName))
{
throw new ArgumentNullException("strProviderName");
}
if (streamPubSub == null)
{
throw new ArgumentNullException("streamPubSub", "StreamPubSub reference should not be null");
}
if (streamQueueBalancer == null)
{
throw new ArgumentNullException("streamQueueBalancer", "IStreamQueueBalancer streamQueueBalancer reference should not be null");
}
queuesToAgentsMap = new Dictionary <QueueId, PersistentStreamPullingAgent>();
streamProviderName = strProviderName;
pubSub = streamPubSub;
this.options = options;
nonReentrancyGuarantor = new AsyncSerialExecutor();
latestRingNotificationSequenceNumber = 0;
latestCommandNumber = 0;
queueBalancer = streamQueueBalancer;
this.streamFilter = streamFilter;
this.adapterFactory = adapterFactory;
this.queueAdapter = queueAdapter ?? throw new ArgumentNullException(nameof(queueAdapter));
queueAdapterCache = adapterFactory.GetQueueAdapterCache();
logger = loggerFactory.CreateLogger($"{GetType().FullName}.{streamProviderName}");
Log(ErrorCode.PersistentStreamPullingManager_01, "Created {0} for Stream Provider {1}.", GetType().Name, streamProviderName);
this.loggerFactory = loggerFactory;
IntValueStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_PULLING_AGENTS, strProviderName), () => queuesToAgentsMap.Count);
}
private async Task Init(CancellationToken token)
{
if (!this.stateManager.PresetState(ProviderState.Initialized))
{
return;
}
this.adapterFactory = this.runtime.ServiceProvider.GetRequiredServiceByName <IQueueAdapterFactory>(this.Name);
this.queueAdapter = await adapterFactory.CreateAdapter();
if (this.pubsubOptions.PubSubType == StreamPubSubType.ExplicitGrainBasedAndImplicit ||
this.pubsubOptions.PubSubType == StreamPubSubType.ExplicitGrainBasedOnly)
{
this.streamSubscriptionManager = this.runtime.ServiceProvider
.GetService <IStreamSubscriptionManagerAdmin>().GetStreamSubscriptionManager(StreamSubscriptionManagerType.ExplicitSubscribeOnly);
}
this.stateManager.CommitState();
}
public RabbitMqAdapterFactory(
string providerName,
IOptionsMonitor <RabbitMqOptions> rmqOptionsAccessor,
IOptionsMonitor <SimpleQueueCacheOptions> cachingOptionsAccessor,
IServiceProvider serviceProvider,
ILoggerFactory loggerFactory,
IRabbitMqStreamQueueMapperFactory streamQueueMapperFactory)
{
if (string.IsNullOrEmpty(providerName))
{
throw new ArgumentNullException(nameof(providerName));
}
if (rmqOptionsAccessor == null)
{
throw new ArgumentNullException(nameof(rmqOptionsAccessor));
}
if (cachingOptionsAccessor == null)
{
throw new ArgumentNullException(nameof(cachingOptionsAccessor));
}
if (serviceProvider == null)
{
throw new ArgumentNullException(nameof(serviceProvider));
}
if (loggerFactory == null)
{
throw new ArgumentNullException(nameof(loggerFactory));
}
if (streamQueueMapperFactory == null)
{
throw new ArgumentNullException(nameof(streamQueueMapperFactory));
}
var rmqOptions = rmqOptionsAccessor.Get(providerName);
var cachingOptions = cachingOptionsAccessor.Get(providerName);
_cache = new SimpleQueueAdapterCache(cachingOptions, providerName, loggerFactory);
_mapper = streamQueueMapperFactory.Get(providerName);
_failureHandler = Task.FromResult <IStreamFailureHandler>(new NoOpStreamDeliveryFailureHandler(false));
var dataAdapter = serviceProvider.GetServiceByName <IQueueDataAdapter <RabbitMqMessage, IEnumerable <IBatchContainer> > >(providerName) ??
RabbitMqDataAdapter.Create(serviceProvider, providerName);
_adapter = new RabbitMqAdapter(rmqOptions, dataAdapter, providerName, _mapper, serviceProvider.GetRequiredServiceByName <IRabbitMqConnectorFactory>(providerName));
}
public async Task <IPersistentStreamPullingManager> InitializePullingAgents(
string streamProviderName,
IQueueAdapterFactory adapterFactory,
IQueueAdapter queueAdapter,
PersistentStreamProviderConfig config)
{
IStreamQueueBalancer queueBalancer = StreamQueueBalancerFactory.Create(config.BalancerType, streamProviderName, this.siloStatusOracle, this.siloDetails.ClusterConfig, this, adapterFactory.GetStreamQueueMapper(), config.SiloMaturityPeriod);
var managerId = GrainId.NewSystemTargetGrainIdByTypeCode(Constants.PULLING_AGENTS_MANAGER_SYSTEM_TARGET_TYPE_CODE);
var manager = new PersistentStreamPullingManager(managerId, streamProviderName, this, this.PubSub(config.PubSubType), adapterFactory, queueBalancer, config);
this.RegisterSystemTarget(manager);
// Init the manager only after it was registered locally.
var pullingAgentManager = manager.AsReference <IPersistentStreamPullingManager>();
// Need to call it as a grain reference though.
await pullingAgentManager.Initialize(queueAdapter.AsImmutable());
return(pullingAgentManager);
}
public async Task StartPullingAgents(
string streamProviderName,
StreamQueueBalancerType balancerType,
IQueueAdapter queueAdapter,
TimeSpan getQueueMsgsTimerPeriod,
TimeSpan initQueueTimeout)
{
IStreamQueueBalancer queueBalancer = StreamQueueBalancerFactory.Create(
balancerType, streamProviderName, Silo.CurrentSilo.LocalSiloStatusOracle, this, queueAdapter.GetStreamQueueMapper());
var managerId = GrainId.NewSystemTargetGrainIdByTypeCode(Constants.PULLING_AGENTS_MANAGER_SYSTEM_TARGET_TYPE_CODE);
var manager = new PersistentStreamPullingManager(managerId, streamProviderName, this, queueBalancer, getQueueMsgsTimerPeriod, initQueueTimeout);
this.RegisterSystemTarget(manager);
// Init the manager only after it was registered locally.
var managerGrainRef = PersistentStreamPullingManagerFactory.Cast(manager.AsReference());
// Need to call it as a grain reference though.
await managerGrainRef.Initialize(queueAdapter.AsImmutable());
}
public RabbitMqAdapterFactory(
string providerName,
RabbitMqOptions rmqOptions,
CachingOptions cachingOptions,
IServiceProvider serviceProvider,
ILoggerFactory loggerFactory)
{
if (string.IsNullOrEmpty(providerName))
{
throw new ArgumentNullException(nameof(providerName));
}
if (rmqOptions == null)
{
throw new ArgumentNullException(nameof(rmqOptions));
}
if (cachingOptions == null)
{
throw new ArgumentNullException(nameof(cachingOptions));
}
if (serviceProvider == null)
{
throw new ArgumentNullException(nameof(serviceProvider));
}
if (loggerFactory == null)
{
throw new ArgumentNullException(nameof(loggerFactory));
}
_cache = new SimpleQueueAdapterCache(new SimpleQueueCacheOptions {
CacheSize = cachingOptions.CacheSize
}, providerName, loggerFactory);
_mapper = new HashRingBasedStreamQueueMapper(new HashRingStreamQueueMapperOptions {
TotalQueueCount = rmqOptions.NumberOfQueues
}, rmqOptions.QueueNamePrefix);
_failureHandler = Task.FromResult <IStreamFailureHandler>(new NoOpStreamDeliveryFailureHandler(false));
var serializer = typeof(TSerializer) == typeof(DefaultBatchContainerSerializer)
? new DefaultBatchContainerSerializer(serviceProvider.GetRequiredService <SerializationManager>())
: (IBatchContainerSerializer) new TSerializer();
_adapter = new RabbitMqAdapter(rmqOptions, cachingOptions, serializer, _mapper, providerName, loggerFactory);
}
internal PersistentStreamPullingAgent(
SystemTargetGrainId id,
string strProviderName,
ILoggerFactory loggerFactory,
IStreamPubSub streamPubSub,
IStreamFilter streamFilter,
QueueId queueId,
StreamPullingAgentOptions options,
SiloAddress siloAddress,
IQueueAdapter queueAdapter,
IQueueAdapterCache queueAdapterCache,
IStreamFailureHandler streamFailureHandler)
: base(id, siloAddress, true, loggerFactory)
{
if (strProviderName == null)
{
throw new ArgumentNullException("runtime", "PersistentStreamPullingAgent: strProviderName should not be null");
}
QueueId = queueId;
streamProviderName = strProviderName;
pubSub = streamPubSub;
this.streamFilter = streamFilter;
pubSubCache = new Dictionary <InternalStreamId, StreamConsumerCollection>();
this.options = options;
this.queueAdapter = queueAdapter ?? throw new ArgumentNullException(nameof(queueAdapter));
this.streamFailureHandler = streamFailureHandler ?? throw new ArgumentNullException(nameof(streamFailureHandler));;
this.queueAdapterCache = queueAdapterCache;
numMessages = 0;
logger = loggerFactory.CreateLogger($"{this.GetType().Namespace}.{streamProviderName}");
logger.LogInformation(
(int)ErrorCode.PersistentStreamPullingAgent_01,
"Created {Name} {Id} for Stream Provider {StreamProvider} on silo {Silo} for Queue {Queue}.",
GetType().Name,
((ISystemTargetBase)this).GrainId.ToString(),
streamProviderName,
Silo,
QueueId.ToStringWithHashCode());
numReadMessagesCounter = CounterStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_READ_MESSAGES, StatisticUniquePostfix));
numSentMessagesCounter = CounterStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_SENT_MESSAGES, StatisticUniquePostfix));
}
public MusicBeeDependencies(ILibraryApiAdapter libraryAdapter,
INowPlayingApiAdapter nowPlayingAdapter,
IOutputApiAdapter outputAdapter,
IPlayerApiAdapter playerAdapter,
IQueueAdapter queueAdapter,
ITrackApiAdapter trackAdapter,
IInvokeHandler invokeHandler,
string baseStoragePath,
string currentVersion)
{
LibraryAdapter = libraryAdapter;
NowPlayingAdapter = nowPlayingAdapter;
OutputAdapter = outputAdapter;
PlayerAdapter = playerAdapter;
QueueAdapter = queueAdapter;
TrackAdapter = trackAdapter;
InvokeHandler = invokeHandler;
BaseStoragePath = baseStoragePath;
CurrentVersion = currentVersion;
}
public Task InitializePullingAgents(
string streamProviderName,
StreamQueueBalancerType balancerType,
StreamPubSubType pubSubType,
IQueueAdapterFactory adapterFactory,
IQueueAdapter queueAdapter,
TimeSpan getQueueMsgsTimerPeriod,
TimeSpan initQueueTimeout,
TimeSpan maxEventDeliveryTime)
{
IStreamQueueBalancer queueBalancer = StreamQueueBalancerFactory.Create(
balancerType, streamProviderName, Silo.CurrentSilo.LocalSiloStatusOracle, Silo.CurrentSilo.OrleansConfig, this, adapterFactory.GetStreamQueueMapper());
var managerId = GrainId.NewSystemTargetGrainIdByTypeCode(Constants.PULLING_AGENTS_MANAGER_SYSTEM_TARGET_TYPE_CODE);
var manager = new PersistentStreamPullingManager(managerId, streamProviderName, this, this.PubSub(pubSubType), adapterFactory, queueBalancer, getQueueMsgsTimerPeriod, initQueueTimeout, maxEventDeliveryTime);
this.RegisterSystemTarget(manager);
// Init the manager only after it was registered locally.
pullingAgentManager = manager.AsReference <IPersistentStreamPullingManager>();
// Need to call it as a grain reference though.
return(pullingAgentManager.Initialize(queueAdapter.AsImmutable()));
}
public QueueAdapter([NotNull] TQueue tQueue)
{
_adapter = tQueue switch
{
IDeque <T> deque => new DequeProxy(deque),
ICircularBuffer <T> buffer => new CircularBufferProxy(buffer),
IHeap <T> heap => new HeapProxy(heap),
Queue <T> queue => new QueueProxy(queue),
IQueue <T> queue => new QueueProxy(queue),
Stack <T> stack => new StackProxy(stack),
IStack <T> stack => new StackProxy(stack),
ConcurrentQueue <T> queue => new ConcurrentQueueProxy(queue),
ConcurrentStack <T> stack => new ConcurrentStackProxy(stack),
BlockingCollection <T> blockingCollection => new BlockingCollectionProxy(blockingCollection),
LinkedList <T> linkedList => new LinkedListProxy(linkedList),
List <T> list => new ListProxy(list),
_ => throw new NotSupportedException()
};
Queue = tQueue;
}
public void Init(IProviderConfiguration config, string providerName, Logger logger, IServiceProvider serviceProvider)
{
if (config == null)
{
throw new ArgumentNullException(nameof(config));
}
if (logger == null)
{
throw new ArgumentNullException(nameof(logger));
}
if (string.IsNullOrEmpty(providerName))
{
throw new ArgumentNullException(nameof(providerName));
}
_options = new RabbitMqStreamProviderOptions(config);
_providerName = providerName;
_cache = new ConcurrentQueueAdapterCache(_options.CacheSize);
_mapper = new HashRingBasedStreamQueueMapper(_options.NumberOfQueues, _options.QueueNamePrefix);
_failureHandler = Task.FromResult <IStreamFailureHandler>(new NoOpStreamDeliveryFailureHandler(false));
_adapter = new RabbitMqAdapter(_options, serviceProvider.GetRequiredService <SerializationManager>(), _mapper, _providerName, logger);
}
public async Task Initialize(Immutable <IQueueAdapter> qAdapter)
{
if (qAdapter.Value == null)
{
throw new ArgumentNullException("qAdapter", "Init: queueAdapter should not be null");
}
Log(ErrorCode.PersistentStreamPullingManager_02, "Init.");
// Remove cast once we cleanup
queueAdapter = qAdapter.Value;
await this.queueBalancer.Initialize(this.adapterFactory.GetStreamQueueMapper());
queueBalancer.SubscribeToQueueDistributionChangeEvents(this);
List <QueueId> myQueues = queueBalancer.GetMyQueues().ToList();
Log(ErrorCode.PersistentStreamPullingManager_03, String.Format("Initialize: I am now responsible for {0} queues: {1}.", myQueues.Count, PrintQueues(myQueues)));
queuePrintTimer = this.RegisterTimer(AsyncTimerCallback, null, QUEUES_PRINT_PERIOD, QUEUES_PRINT_PERIOD);
managerState = RunState.Initialized;
}
public Task Initialize(Immutable <IQueueAdapter> qAdapter)
{
if (qAdapter.Value == null)
{
throw new ArgumentNullException("qAdapter", "Init: queueAdapter should not be null");
}
logger.Info((int)ErrorCode.PersistentStreamPullingManager_02, "Init.");
// Remove cast once we cleanup
queueAdapter = qAdapter.Value;
var meAsQueueBalanceListener = this.AsReference <IStreamQueueBalanceListener>();
queueBalancer.SubscribeToQueueDistributionChangeEvents(meAsQueueBalanceListener);
List <QueueId> myQueues = queueBalancer.GetMyQueues().ToList();
logger.Info((int)ErrorCode.PersistentStreamPullingManager_03, PrintMyState(myQueues));
return(AddNewQueues(myQueues, true));
}
public Task Initialize(Immutable <IQueueAdapter> qAdapter)
{
if (qAdapter.Value == null)
{
throw new ArgumentNullException("qAdapter", "Init: queueAdapter should not be null");
}
logger.Info((int)ErrorCode.PersistentStreamPullingManager_02, "Init.");
// Remove cast once we cleanup
queueAdapter = qAdapter.Value;
var meAsQueueBalanceListener = this.AsReference <IStreamQueueBalanceListener>();
queueBalancer.SubscribeToQueueDistributionChangeEvents(meAsQueueBalanceListener);
List <QueueId> myQueues = queueBalancer.GetMyQueues().ToList();
logger.Info((int)ErrorCode.PersistentStreamPullingManager_03, String.Format("I am now responsible for {0} queues: {1}.", myQueues.Count, PrintQueues(myQueues)));
managerState = PersistentStreamProviderState.Initialized;
return(TaskDone.Done);
}
public Task Initialize(Immutable<IQueueAdapter> qAdapter)
{
if (qAdapter.Value == null) throw new ArgumentNullException("qAdapter", "Init: queueAdapter should not be null");
logger.Info((int)ErrorCode.PersistentStreamPullingManager_02, "Init.");
// Remove cast once we cleanup
queueAdapter = qAdapter.Value;
var meAsQueueBalanceListener = this.AsReference<IStreamQueueBalanceListener>();
queueBalancer.SubscribeToQueueDistributionChangeEvents(meAsQueueBalanceListener);
List<QueueId> myQueues = queueBalancer.GetMyQueues().ToList();
logger.Info((int)ErrorCode.PersistentStreamPullingManager_03, PrintMyState(myQueues));
return AddNewQueues(myQueues, true);
}
public Task Initialize(Immutable<IQueueAdapter> qAdapter)
{
if (qAdapter.Value == null) throw new ArgumentNullException("qAdapter", "Init: queueAdapter should not be null");
Log(ErrorCode.PersistentStreamPullingManager_02, "Init.");
// Remove cast once we cleanup
queueAdapter = qAdapter.Value;
var meAsQueueBalanceListener = this.AsReference<IStreamQueueBalanceListener>();
queueBalancer.SubscribeToQueueDistributionChangeEvents(meAsQueueBalanceListener);
List<QueueId> myQueues = queueBalancer.GetMyQueues().ToList();
Log(ErrorCode.PersistentStreamPullingManager_03, String.Format("Initialize: I am now responsible for {0} queues: {1}.", myQueues.Count, PrintQueues(myQueues)));
managerState = PersistentStreamProviderState.Initialized;
return TaskDone.Done;
}
/// <summary>
/// Take responsibility for a new queues that was assigned to me via a new range.
/// We first store the new queue in our internal data structure, try to initialize it and start a pumping timer.
/// ERROR HANDLING:
/// The resposibility to handle initializatoion and shutdown failures is inside the INewQueueAdapterReceiver code.
/// The agent will call Initialize once and log an error. It will not call initiliaze again.
/// The receiver itself may attempt later to recover from this error and do initialization again.
/// The agent will assume initialization has succeeded and will subsequently start calling pumping receive.
/// Same applies to shutdown.
/// </summary>
/// <param name="qAdapter"></param>
/// <param name="queueAdapterCache"></param>
/// <param name="failureHandler"></param>
/// <returns></returns>
public async Task Initialize(Immutable<IQueueAdapter> qAdapter, Immutable<IQueueAdapterCache> queueAdapterCache, Immutable<IStreamFailureHandler> failureHandler)
{
if (qAdapter.Value == null) throw new ArgumentNullException("qAdapter", "Init: queueAdapter should not be null");
if (failureHandler.Value == null) throw new ArgumentNullException("failureHandler", "Init: streamDeliveryFailureHandler should not be null");
logger.Info((int)ErrorCode.PersistentStreamPullingAgent_02, "Init of {0} {1} on silo {2} for queue {3}.",
GetType().Name, GrainId.ToDetailedString(), Silo, QueueId.ToStringWithHashCode());
// Remove cast once we cleanup
queueAdapter = qAdapter.Value;
streamFailureHandler = failureHandler.Value;
lastTimeCleanedPubSubCache = DateTime.UtcNow;
try
{
receiver = queueAdapter.CreateReceiver(QueueId);
}
catch (Exception exc)
{
logger.Error((int)ErrorCode.PersistentStreamPullingAgent_02, "Exception while calling IQueueAdapter.CreateNewReceiver.", exc);
return;
}
try
{
if (queueAdapterCache.Value != null)
{
queueCache = queueAdapterCache.Value.CreateQueueCache(QueueId);
}
}
catch (Exception exc)
{
logger.Error((int)ErrorCode.PersistentStreamPullingAgent_23, "Exception while calling IQueueAdapterCache.CreateQueueCache.", exc);
return;
}
try
{
var task = OrleansTaskExtentions.SafeExecute(() => receiver.Initialize(config.InitQueueTimeout));
task = task.LogException(logger, ErrorCode.PersistentStreamPullingAgent_03, String.Format("QueueAdapterReceiver {0} failed to Initialize.", QueueId.ToStringWithHashCode()));
await task;
}
catch
{
// Just ignore this exception and proceed as if Initialize has succeeded.
// We already logged individual exceptions for individual calls to Initialize. No need to log again.
}
// Setup a reader for a new receiver.
// Even if the receiver failed to initialise, treat it as OK and start pumping it. It's receiver responsibility to retry initialization.
var randomTimerOffset = safeRandom.NextTimeSpan(config.GetQueueMsgsTimerPeriod);
timer = base.RegisterTimer(AsyncTimerCallback, QueueId, randomTimerOffset, config.GetQueueMsgsTimerPeriod);
logger.Info((int) ErrorCode.PersistentStreamPullingAgent_04, "Taking queue {0} under my responsibility.", QueueId.ToStringWithHashCode());
}
private void InitializeInternal(IQueueAdapter qAdapter, IQueueAdapterCache queueAdapterCache, IStreamFailureHandler failureHandler)
{
logger.Info(ErrorCode.PersistentStreamPullingAgent_02, "Init of {0} {1} on silo {2} for queue {3}.",
GetType().Name, GrainId.ToDetailedString(), Silo, QueueId.ToStringWithHashCode());
// Remove cast once we cleanup
queueAdapter = qAdapter;
streamFailureHandler = failureHandler;
lastTimeCleanedPubSubCache = DateTime.UtcNow;
try
{
receiver = queueAdapter.CreateReceiver(QueueId);
}
catch (Exception exc)
{
logger.Error(ErrorCode.PersistentStreamPullingAgent_02, "Exception while calling IQueueAdapter.CreateNewReceiver.", exc);
throw;
}
try
{
if (queueAdapterCache != null)
{
queueCache = queueAdapterCache.CreateQueueCache(QueueId);
}
}
catch (Exception exc)
{
logger.Error(ErrorCode.PersistentStreamPullingAgent_23, "Exception while calling IQueueAdapterCache.CreateQueueCache.", exc);
throw;
}
try
{
receiverInitTask = OrleansTaskExtentions.SafeExecute(() => receiver.Initialize(config.InitQueueTimeout))
.LogException(logger, ErrorCode.PersistentStreamPullingAgent_03, $"QueueAdapterReceiver {QueueId.ToStringWithHashCode()} failed to Initialize.");
receiverInitTask.Ignore();
}
catch
{
// Just ignore this exception and proceed as if Initialize has succeeded.
// We already logged individual exceptions for individual calls to Initialize. No need to log again.
}
// Setup a reader for a new receiver.
// Even if the receiver failed to initialise, treat it as OK and start pumping it. It's receiver responsibility to retry initialization.
var randomTimerOffset = safeRandom.NextTimeSpan(config.GetQueueMsgsTimerPeriod);
timer = RegisterTimer(AsyncTimerCallback, QueueId, randomTimerOffset, config.GetQueueMsgsTimerPeriod);
logger.Info((int)ErrorCode.PersistentStreamPullingAgent_04, "Taking queue {0} under my responsibility.", QueueId.ToStringWithHashCode());
}
