public async Task <IPersistentStreamPullingManager> InitializePullingAgents( string streamProviderName, IQueueAdapterFactory adapterFactory, IQueueAdapter queueAdapter) { IStreamQueueBalancer queueBalancer = CreateQueueBalancer(streamProviderName); var managerId = LegacyGrainId.NewSystemTargetGrainIdByTypeCode(Constants.PULLING_AGENTS_MANAGER_SYSTEM_TARGET_TYPE_CODE); var pubsubOptions = this.ServiceProvider.GetOptionsByName <StreamPubSubOptions>(streamProviderName); var pullingAgentOptions = this.ServiceProvider.GetOptionsByName <StreamPullingAgentOptions>(streamProviderName); var manager = new PersistentStreamPullingManager( managerId, streamProviderName, this, this.PubSub(pubsubOptions.PubSubType), adapterFactory, queueBalancer, pullingAgentOptions, this.loggerFactory, this.siloDetails.SiloAddress); this.RegisterSystemTarget(manager); // Init the manager only after it was registered locally. var pullingAgentManager = manager.AsReference <IPersistentStreamPullingManager>(); // Need to call it as a grain reference though. await pullingAgentManager.Initialize(queueAdapter.AsImmutable()); return(pullingAgentManager); }
public Publisher(ILogController logController, IOptions <ConnectionStrings> connectionStrings, IOptions <AzureServiceBusConfiguration> serviceBusConfiguration) : base(logController, connectionStrings, serviceBusConfiguration) { _queueManager = this.BuildDefaultQueueAdapter(); logger = this.BuildLogger(); _queueManager.Initialize(); this.logController = logController; }
public async Task Init(string name, IProviderRuntime providerUtilitiesManager, IProviderConfiguration config) { if (!stateManager.PresetState(ProviderState.Initialized)) { return; } if (String.IsNullOrEmpty(name)) { throw new ArgumentNullException("name"); } if (providerUtilitiesManager == null) { throw new ArgumentNullException("providerUtilitiesManager"); } if (config == null) { throw new ArgumentNullException("config"); } Name = name; providerRuntime = (IStreamProviderRuntime)providerUtilitiesManager; logger = providerRuntime.GetLogger(this.GetType().Name); adapterFactory = new TAdapterFactory(); // Temporary change, but we need GrainFactory inside ServiceProvider for now, // so will change it back as soon as we have an action item to add GrainFactory to ServiceProvider. adapterFactory.Init(config, Name, logger, new GrainFactoryServiceProvider(providerRuntime)); queueAdapter = await adapterFactory.CreateAdapter(); myConfig = new PersistentStreamProviderConfig(config); this.providerConfig = config; this.serializationManager = this.providerRuntime.ServiceProvider.GetRequiredService <SerializationManager>(); this.runtimeClient = this.providerRuntime.ServiceProvider.GetRequiredService <IRuntimeClient>(); if (this.myConfig.PubSubType == StreamPubSubType.ExplicitGrainBasedAndImplicit || this.myConfig.PubSubType == StreamPubSubType.ExplicitGrainBasedOnly) { this.streamSubscriptionManager = this.providerRuntime.ServiceProvider .GetService <IStreamSubscriptionManagerAdmin>().GetStreamSubscriptionManager(StreamSubscriptionManagerType.ExplicitSubscribeOnly); } string startup; if (config.Properties.TryGetValue(StartupStatePropertyName, out startup)) { if (!Enum.TryParse(startup, true, out startupState)) { throw new ArgumentException( String.Format("Unsupported value '{0}' for configuration parameter {1} of stream provider {2}.", startup, StartupStatePropertyName, config.Name)); } } else { startupState = StartupStateDefaultValue; } logger.Info("Initialized PersistentStreamProvider<{0}> with name {1}, Adapter {2} and config {3}, {4} = {5}.", typeof(TAdapterFactory).Name, Name, queueAdapter.Name, myConfig, StartupStatePropertyName, startupState); stateManager.CommitState(); }
public async Task <IPersistentStreamPullingManager> InitializePullingAgents( string streamProviderName, IQueueAdapterFactory adapterFactory, IQueueAdapter queueAdapter) { IStreamQueueBalancer queueBalancer = CreateQueueBalancer(streamProviderName); var managerId = SystemTargetGrainId.Create(Constants.StreamPullingAgentManagerType, this.siloDetails.SiloAddress, streamProviderName); var pubsubOptions = this.ServiceProvider.GetOptionsByName <StreamPubSubOptions>(streamProviderName); var pullingAgentOptions = this.ServiceProvider.GetOptionsByName <StreamPullingAgentOptions>(streamProviderName); var filter = this.ServiceProvider.GetServiceByName <IStreamFilter>(streamProviderName) ?? new NoOpStreamFilter(); var manager = new PersistentStreamPullingManager( managerId, streamProviderName, this.PubSub(pubsubOptions.PubSubType), adapterFactory, queueBalancer, filter, pullingAgentOptions, this.loggerFactory, this.siloDetails.SiloAddress, queueAdapter); var catalog = this.ServiceProvider.GetRequiredService <Catalog>(); catalog.RegisterSystemTarget(manager); // Init the manager only after it was registered locally. var pullingAgentManager = manager.AsReference <IPersistentStreamPullingManager>(); // Need to call it as a grain reference though. await pullingAgentManager.Initialize(); return(pullingAgentManager); }
/// <summary> /// Initializes a new instance of the <see cref="Publisher<T>"/> class. /// </summary> /// <param name="queueManager">The queue manager.</param> /// <param name="logger">The logger.</param> public Publisher(IQueueAdapter queueManager, IBusLogger logger, ILogController logController, IOptions <ConnectionStrings> connectionStrings, IOptions <AzureServiceBusConfiguration> serviceBusConfiguration) : base(logController, connectionStrings, serviceBusConfiguration) { this.logger = logger; this._queueManager = queueManager; queueManager.Initialize(); this.logController = logController; }
/// <summary> /// Take responsibility for a new queues that was assigned to me via a new range. /// We first store the new queue in our internal data structure, try to initialize it and start a pumping timer. /// ERROR HANDLING: /// The resposibility to handle initializatoion and shutdown failures is inside the INewQueueAdapterReceiver code. /// The agent will call Initialize once and log an error. It will not call initiliaze again. /// The receiver itself may attempt later to recover from this error and do initialization again. /// The agent will assume initialization has succeeded and will subsequently start calling pumping receive. /// Same applies to shutdown. /// </summary> /// <param name="qAdapter"></param> /// <param name="queueAdapterCache"></param> /// <param name="failureHandler"></param> /// <returns></returns> public async Task Initialize(Immutable <IQueueAdapter> qAdapter, Immutable <IQueueAdapterCache> queueAdapterCache, Immutable <IStreamFailureHandler> failureHandler) { if (qAdapter.Value == null) { throw new ArgumentNullException("qAdapter", "Init: queueAdapter should not be null"); } if (failureHandler.Value == null) { throw new ArgumentNullException("failureHandler", "Init: streamDeliveryFailureHandler should not be null"); } logger.Info((int)ErrorCode.PersistentStreamPullingAgent_02, "Init of {0} {1} on silo {2} for queue {3}.", GetType().Name, GrainId.ToDetailedString(), Silo, QueueId.ToStringWithHashCode()); // Remove cast once we cleanup queueAdapter = qAdapter.Value; streamFailureHandler = failureHandler.Value; try { receiver = queueAdapter.CreateReceiver(QueueId); } catch (Exception exc) { logger.Error((int)ErrorCode.PersistentStreamPullingAgent_02, String.Format("Exception while calling IQueueAdapter.CreateNewReceiver."), exc); return; } try { if (queueAdapterCache.Value != null) { queueCache = queueAdapterCache.Value.CreateQueueCache(QueueId); } } catch (Exception exc) { logger.Error((int)ErrorCode.PersistentStreamPullingAgent_23, String.Format("Exception while calling IQueueAdapterCache.CreateQueueCache."), exc); return; } try { var task = OrleansTaskExtentions.SafeExecute(() => receiver.Initialize(initQueueTimeout)); task = task.LogException(logger, ErrorCode.PersistentStreamPullingAgent_03, String.Format("QueueAdapterReceiver {0} failed to Initialize.", QueueId.ToStringWithHashCode())); await task; } catch { // Just ignore this exception and proceed as if Initialize has succeeded. // We already logged individual exceptions for individual calls to Initialize. No need to log again. } // Setup a reader for a new receiver. // Even if the receiver failed to initialise, treat it as OK and start pumping it. It's receiver responsibility to retry initialization. var randomTimerOffset = safeRandom.NextTimeSpan(queueGetPeriod); timer = providerRuntime.RegisterTimer(AsyncTimerCallback, QueueId, randomTimerOffset, queueGetPeriod); logger.Info((int)ErrorCode.PersistentStreamPullingAgent_04, "Taking queue {0} under my responsibility.", QueueId.ToStringWithHashCode()); }
public async Task Init(string name, IProviderRuntime providerUtilitiesManager, IProviderConfiguration config) { if (String.IsNullOrEmpty(name)) { throw new ArgumentNullException("name"); } if (providerUtilitiesManager == null) { throw new ArgumentNullException("providerUtilitiesManager"); } if (config == null) { throw new ArgumentNullException("config"); } Name = name; providerRuntime = (IStreamProviderRuntime)providerUtilitiesManager; logger = providerRuntime.GetLogger(this.GetType().Name); IQueueAdapterFactory factory = new TAdapterFactory(); factory.Init(config, Name, logger); queueAdapter = await factory.CreateAdapter(); string timePeriod; if (!config.Properties.TryGetValue(GET_QUEUE_MESSAGES_TIMER_PERIOD, out timePeriod)) { getQueueMsgsTimerPeriod = DEFAULT_GET_QUEUE_MESSAGES_TIMER_PERIOD; } else { getQueueMsgsTimerPeriod = ConfigUtilities.ParseTimeSpan(timePeriod, "Invalid time value for the " + GET_QUEUE_MESSAGES_TIMER_PERIOD + " property in the provider config values."); } string timeout; if (!config.Properties.TryGetValue(INIT_QUEUE_TIMEOUT, out timeout)) { initQueueTimeout = DEFAULT_INIT_QUEUE_TIMEOUT; } else { initQueueTimeout = ConfigUtilities.ParseTimeSpan(timeout, "Invalid time value for the " + INIT_QUEUE_TIMEOUT + " property in the provider config values."); } string balanceTypeString; balancerType = !config.Properties.TryGetValue(QUEUE_BALANCER_TYPE, out balanceTypeString) ? DEFAULT_STREAM_QUEUE_BALANCER_TYPE : (StreamQueueBalancerType)Enum.Parse(typeof(StreamQueueBalancerType), balanceTypeString); logger.Info("Initialized PersistentStreamProvider<{0}> with name {1}, {2} = {3}, {4} = {5} and with Adapter {6}.", typeof(TAdapterFactory).Name, Name, GET_QUEUE_MESSAGES_TIMER_PERIOD, getQueueMsgsTimerPeriod, INIT_QUEUE_TIMEOUT, this.initQueueTimeout, queueAdapter.Name); }
/// <summary> /// Builds the publisher. /// </summary> /// <returns></returns> public Publisher <T> BuildPublisher() { IQueueAdapter queueAdapter = BuildDefaultQueueAdapter(); IBusLogger logger = BuildLogger(); var publisher = new Publisher <T>(queueAdapter, logger, this.logController, this.connectionStrings, this.serviceBusConfiguration); return(publisher); }
/// <summary> /// Sends the message. /// </summary> /// <param name="message">The message.</param> public virtual void SendMessage(T message, string azureServiceBusTopicName) { _queueManager = BuildAzureQueueAdapterForTopic(azureServiceBusTopicName); var envelope = new MessageEnvelope(message); SendEnvelope(envelope); }
public Task StartPullingAgents( string streamProviderName, StreamQueueBalancerType balancerType, IQueueAdapter queueAdapter, TimeSpan getQueueMsgsTimerPeriod, TimeSpan initQueueTimeout) { return(TaskDone.Done); }
private void InitializeInternal(IQueueAdapter qAdapter, IQueueAdapterCache queueAdapterCache, IStreamFailureHandler failureHandler) { logger.Info(ErrorCode.PersistentStreamPullingAgent_02, "Init of {0} {1} on silo {2} for queue {3}.", GetType().Name, ((ISystemTargetBase)this).GrainId.ToDetailedString(), Silo, QueueId.ToStringWithHashCode()); // Remove cast once we cleanup queueAdapter = qAdapter; streamFailureHandler = failureHandler; lastTimeCleanedPubSubCache = DateTime.UtcNow; try { receiver = queueAdapter.CreateReceiver(QueueId); } catch (Exception exc) { logger.Error(ErrorCode.PersistentStreamPullingAgent_02, "Exception while calling IQueueAdapter.CreateNewReceiver.", exc); throw; } try { if (queueAdapterCache != null) { queueCache = queueAdapterCache.CreateQueueCache(QueueId); } } catch (Exception exc) { logger.Error(ErrorCode.PersistentStreamPullingAgent_23, "Exception while calling IQueueAdapterCache.CreateQueueCache.", exc); throw; } try { receiverInitTask = OrleansTaskExtentions.SafeExecute(() => receiver.Initialize(config.InitQueueTimeout)) .LogException(logger, ErrorCode.PersistentStreamPullingAgent_03, $"QueueAdapterReceiver {QueueId.ToStringWithHashCode()} failed to Initialize."); receiverInitTask.Ignore(); } catch { // Just ignore this exception and proceed as if Initialize has succeeded. // We already logged individual exceptions for individual calls to Initialize. No need to log again. } // Setup a reader for a new receiver. // Even if the receiver failed to initialise, treat it as OK and start pumping it. It's receiver responsibility to retry initialization. var randomTimerOffset = safeRandom.NextTimeSpan(config.GetQueueMsgsTimerPeriod); timer = RegisterTimer(AsyncTimerCallback, QueueId, randomTimerOffset, config.GetQueueMsgsTimerPeriod); IntValueStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_PUBSUB_CACHE_SIZE, StatisticUniquePostfix), () => pubSubCache.Count); logger.Info((int)ErrorCode.PersistentStreamPullingAgent_04, "Taking queue {0} under my responsibility.", QueueId.ToStringWithHashCode()); }
public ImagesService(IImagesRepository imagesRepository, IFileStorageAdapter storageAdapter, IQueueAdapter queueAdapter, ICompoundImagesRepository compoundImagesRepository, ICompoundImageMappingsRepository compoundImageMappingsRepository, IMapsAnalyser mapsAnalyser, IMapsRepository mapsRepository, IConfiguration configuration) { _imagesRepository = imagesRepository; _queueAdapter = queueAdapter; _storageAdapter = storageAdapter; _compoundImagesRepository = compoundImagesRepository; _compoundImageMappingsRepository = compoundImageMappingsRepository; _mapsAnalyser = mapsAnalyser; _mapsRepository = mapsRepository; _storageContainerName = configuration.GetValue <string>("Azure:CloudBlobImageContainerName"); }
internal PersistentStreamProducer(StreamImpl <T> stream, IStreamProviderRuntime providerUtilities, IQueueAdapter queueAdapter, bool isRewindable) { this.stream = stream; this.queueAdapter = queueAdapter; IsRewindable = isRewindable; var logger = providerUtilities.GetLogger(this.GetType().Name); if (logger.IsVerbose) { logger.Verbose("Created PersistentStreamProducer for stream {0}, of type {1}, and with Adapter: {2}.", stream.ToString(), typeof(T), this.queueAdapter.Name); } }
public async Task Init(string name, IProviderRuntime providerUtilitiesManager, IProviderConfiguration config) { if (!stateManager.PresetState(ProviderState.Initialized)) { return; } if (String.IsNullOrEmpty(name)) { throw new ArgumentNullException("name"); } if (providerUtilitiesManager == null) { throw new ArgumentNullException("providerUtilitiesManager"); } if (config == null) { throw new ArgumentNullException("config"); } Name = name; providerRuntime = (IStreamProviderRuntime)providerUtilitiesManager; logger = providerRuntime.GetLogger(this.GetType().Name); adapterFactory = new TAdapterFactory(); adapterFactory.Init(config, Name, logger, providerRuntime.ServiceProvider); queueAdapter = await adapterFactory.CreateAdapter(); myConfig = new PersistentStreamProviderConfig(config); string startup; if (config.Properties.TryGetValue(StartupStatePropertyName, out startup)) { if (!Enum.TryParse(startup, true, out startupState)) { throw new ArgumentException( String.Format("Unsupported value '{0}' for configuration parameter {1} of stream provider {2}.", startup, StartupStatePropertyName, config.Name)); } } else { startupState = StartupStateDefaultValue; } logger.Info("Initialized PersistentStreamProvider<{0}> with name {1}, Adapter {2} and config {3}, {4} = {5}.", typeof(TAdapterFactory).Name, Name, queueAdapter.Name, myConfig, StartupStatePropertyName, startupState); stateManager.CommitState(); }
internal PersistentStreamProducer(StreamImpl <T> stream, IStreamProviderRuntime providerUtilities, IQueueAdapter queueAdapter, bool isRewindable, DeepCopier deepCopier) { this.stream = stream; this.queueAdapter = queueAdapter; this.deepCopier = deepCopier; IsRewindable = isRewindable; var logger = providerUtilities.ServiceProvider.GetRequiredService <ILogger <PersistentStreamProducer <T> > >(); if (logger.IsEnabled(LogLevel.Debug)) { logger.LogDebug("Created PersistentStreamProducer for stream {StreamId}, of type {ElementType}, and with Adapter: {QueueAdapterName}.", stream.ToString(), typeof(T), this.queueAdapter.Name); } }
internal PersistentStreamProducer(StreamImpl <T> stream, IStreamProviderRuntime providerUtilities, IQueueAdapter queueAdapter, bool isRewindable, SerializationManager serializationManager) { this.stream = stream; this.queueAdapter = queueAdapter; this.serializationManager = serializationManager; IsRewindable = isRewindable; var logger = providerUtilities.ServiceProvider.GetRequiredService <ILoggerFactory>().CreateLogger(this.GetType().Name); if (logger.IsEnabled(LogLevel.Debug)) { logger.Debug("Created PersistentStreamProducer for stream {0}, of type {1}, and with Adapter: {2}.", stream.ToString(), typeof(T), this.queueAdapter.Name); } }
internal PersistentStreamPullingManager( SystemTargetGrainId managerId, string strProviderName, IStreamPubSub streamPubSub, IQueueAdapterFactory adapterFactory, IStreamQueueBalancer streamQueueBalancer, IStreamFilter streamFilter, StreamPullingAgentOptions options, ILoggerFactory loggerFactory, SiloAddress siloAddress, IQueueAdapter queueAdapter) : base(managerId, siloAddress, lowPriority: false, loggerFactory) { if (string.IsNullOrWhiteSpace(strProviderName)) { throw new ArgumentNullException("strProviderName"); } if (streamPubSub == null) { throw new ArgumentNullException("streamPubSub", "StreamPubSub reference should not be null"); } if (streamQueueBalancer == null) { throw new ArgumentNullException("streamQueueBalancer", "IStreamQueueBalancer streamQueueBalancer reference should not be null"); } queuesToAgentsMap = new Dictionary <QueueId, PersistentStreamPullingAgent>(); streamProviderName = strProviderName; pubSub = streamPubSub; this.options = options; nonReentrancyGuarantor = new AsyncSerialExecutor(); latestRingNotificationSequenceNumber = 0; latestCommandNumber = 0; queueBalancer = streamQueueBalancer; this.streamFilter = streamFilter; this.adapterFactory = adapterFactory; this.queueAdapter = queueAdapter ?? throw new ArgumentNullException(nameof(queueAdapter)); queueAdapterCache = adapterFactory.GetQueueAdapterCache(); logger = loggerFactory.CreateLogger($"{GetType().FullName}.{streamProviderName}"); Log(ErrorCode.PersistentStreamPullingManager_01, "Created {0} for Stream Provider {1}.", GetType().Name, streamProviderName); this.loggerFactory = loggerFactory; IntValueStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_PULLING_AGENTS, strProviderName), () => queuesToAgentsMap.Count); }
private async Task Init(CancellationToken token) { if (!this.stateManager.PresetState(ProviderState.Initialized)) { return; } this.adapterFactory = this.runtime.ServiceProvider.GetRequiredServiceByName <IQueueAdapterFactory>(this.Name); this.queueAdapter = await adapterFactory.CreateAdapter(); if (this.pubsubOptions.PubSubType == StreamPubSubType.ExplicitGrainBasedAndImplicit || this.pubsubOptions.PubSubType == StreamPubSubType.ExplicitGrainBasedOnly) { this.streamSubscriptionManager = this.runtime.ServiceProvider .GetService <IStreamSubscriptionManagerAdmin>().GetStreamSubscriptionManager(StreamSubscriptionManagerType.ExplicitSubscribeOnly); } this.stateManager.CommitState(); }
public RabbitMqAdapterFactory( string providerName, IOptionsMonitor <RabbitMqOptions> rmqOptionsAccessor, IOptionsMonitor <SimpleQueueCacheOptions> cachingOptionsAccessor, IServiceProvider serviceProvider, ILoggerFactory loggerFactory, IRabbitMqStreamQueueMapperFactory streamQueueMapperFactory) { if (string.IsNullOrEmpty(providerName)) { throw new ArgumentNullException(nameof(providerName)); } if (rmqOptionsAccessor == null) { throw new ArgumentNullException(nameof(rmqOptionsAccessor)); } if (cachingOptionsAccessor == null) { throw new ArgumentNullException(nameof(cachingOptionsAccessor)); } if (serviceProvider == null) { throw new ArgumentNullException(nameof(serviceProvider)); } if (loggerFactory == null) { throw new ArgumentNullException(nameof(loggerFactory)); } if (streamQueueMapperFactory == null) { throw new ArgumentNullException(nameof(streamQueueMapperFactory)); } var rmqOptions = rmqOptionsAccessor.Get(providerName); var cachingOptions = cachingOptionsAccessor.Get(providerName); _cache = new SimpleQueueAdapterCache(cachingOptions, providerName, loggerFactory); _mapper = streamQueueMapperFactory.Get(providerName); _failureHandler = Task.FromResult <IStreamFailureHandler>(new NoOpStreamDeliveryFailureHandler(false)); var dataAdapter = serviceProvider.GetServiceByName <IQueueDataAdapter <RabbitMqMessage, IEnumerable <IBatchContainer> > >(providerName) ?? RabbitMqDataAdapter.Create(serviceProvider, providerName); _adapter = new RabbitMqAdapter(rmqOptions, dataAdapter, providerName, _mapper, serviceProvider.GetRequiredServiceByName <IRabbitMqConnectorFactory>(providerName)); }
public async Task <IPersistentStreamPullingManager> InitializePullingAgents( string streamProviderName, IQueueAdapterFactory adapterFactory, IQueueAdapter queueAdapter, PersistentStreamProviderConfig config) { IStreamQueueBalancer queueBalancer = StreamQueueBalancerFactory.Create(config.BalancerType, streamProviderName, this.siloStatusOracle, this.siloDetails.ClusterConfig, this, adapterFactory.GetStreamQueueMapper(), config.SiloMaturityPeriod); var managerId = GrainId.NewSystemTargetGrainIdByTypeCode(Constants.PULLING_AGENTS_MANAGER_SYSTEM_TARGET_TYPE_CODE); var manager = new PersistentStreamPullingManager(managerId, streamProviderName, this, this.PubSub(config.PubSubType), adapterFactory, queueBalancer, config); this.RegisterSystemTarget(manager); // Init the manager only after it was registered locally. var pullingAgentManager = manager.AsReference <IPersistentStreamPullingManager>(); // Need to call it as a grain reference though. await pullingAgentManager.Initialize(queueAdapter.AsImmutable()); return(pullingAgentManager); }
public async Task StartPullingAgents( string streamProviderName, StreamQueueBalancerType balancerType, IQueueAdapter queueAdapter, TimeSpan getQueueMsgsTimerPeriod, TimeSpan initQueueTimeout) { IStreamQueueBalancer queueBalancer = StreamQueueBalancerFactory.Create( balancerType, streamProviderName, Silo.CurrentSilo.LocalSiloStatusOracle, this, queueAdapter.GetStreamQueueMapper()); var managerId = GrainId.NewSystemTargetGrainIdByTypeCode(Constants.PULLING_AGENTS_MANAGER_SYSTEM_TARGET_TYPE_CODE); var manager = new PersistentStreamPullingManager(managerId, streamProviderName, this, queueBalancer, getQueueMsgsTimerPeriod, initQueueTimeout); this.RegisterSystemTarget(manager); // Init the manager only after it was registered locally. var managerGrainRef = PersistentStreamPullingManagerFactory.Cast(manager.AsReference()); // Need to call it as a grain reference though. await managerGrainRef.Initialize(queueAdapter.AsImmutable()); }
public RabbitMqAdapterFactory( string providerName, RabbitMqOptions rmqOptions, CachingOptions cachingOptions, IServiceProvider serviceProvider, ILoggerFactory loggerFactory) { if (string.IsNullOrEmpty(providerName)) { throw new ArgumentNullException(nameof(providerName)); } if (rmqOptions == null) { throw new ArgumentNullException(nameof(rmqOptions)); } if (cachingOptions == null) { throw new ArgumentNullException(nameof(cachingOptions)); } if (serviceProvider == null) { throw new ArgumentNullException(nameof(serviceProvider)); } if (loggerFactory == null) { throw new ArgumentNullException(nameof(loggerFactory)); } _cache = new SimpleQueueAdapterCache(new SimpleQueueCacheOptions { CacheSize = cachingOptions.CacheSize }, providerName, loggerFactory); _mapper = new HashRingBasedStreamQueueMapper(new HashRingStreamQueueMapperOptions { TotalQueueCount = rmqOptions.NumberOfQueues }, rmqOptions.QueueNamePrefix); _failureHandler = Task.FromResult <IStreamFailureHandler>(new NoOpStreamDeliveryFailureHandler(false)); var serializer = typeof(TSerializer) == typeof(DefaultBatchContainerSerializer) ? new DefaultBatchContainerSerializer(serviceProvider.GetRequiredService <SerializationManager>()) : (IBatchContainerSerializer) new TSerializer(); _adapter = new RabbitMqAdapter(rmqOptions, cachingOptions, serializer, _mapper, providerName, loggerFactory); }
internal PersistentStreamPullingAgent( SystemTargetGrainId id, string strProviderName, ILoggerFactory loggerFactory, IStreamPubSub streamPubSub, IStreamFilter streamFilter, QueueId queueId, StreamPullingAgentOptions options, SiloAddress siloAddress, IQueueAdapter queueAdapter, IQueueAdapterCache queueAdapterCache, IStreamFailureHandler streamFailureHandler) : base(id, siloAddress, true, loggerFactory) { if (strProviderName == null) { throw new ArgumentNullException("runtime", "PersistentStreamPullingAgent: strProviderName should not be null"); } QueueId = queueId; streamProviderName = strProviderName; pubSub = streamPubSub; this.streamFilter = streamFilter; pubSubCache = new Dictionary <InternalStreamId, StreamConsumerCollection>(); this.options = options; this.queueAdapter = queueAdapter ?? throw new ArgumentNullException(nameof(queueAdapter)); this.streamFailureHandler = streamFailureHandler ?? throw new ArgumentNullException(nameof(streamFailureHandler));; this.queueAdapterCache = queueAdapterCache; numMessages = 0; logger = loggerFactory.CreateLogger($"{this.GetType().Namespace}.{streamProviderName}"); logger.LogInformation( (int)ErrorCode.PersistentStreamPullingAgent_01, "Created {Name} {Id} for Stream Provider {StreamProvider} on silo {Silo} for Queue {Queue}.", GetType().Name, ((ISystemTargetBase)this).GrainId.ToString(), streamProviderName, Silo, QueueId.ToStringWithHashCode()); numReadMessagesCounter = CounterStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_READ_MESSAGES, StatisticUniquePostfix)); numSentMessagesCounter = CounterStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_SENT_MESSAGES, StatisticUniquePostfix)); }
public MusicBeeDependencies(ILibraryApiAdapter libraryAdapter, INowPlayingApiAdapter nowPlayingAdapter, IOutputApiAdapter outputAdapter, IPlayerApiAdapter playerAdapter, IQueueAdapter queueAdapter, ITrackApiAdapter trackAdapter, IInvokeHandler invokeHandler, string baseStoragePath, string currentVersion) { LibraryAdapter = libraryAdapter; NowPlayingAdapter = nowPlayingAdapter; OutputAdapter = outputAdapter; PlayerAdapter = playerAdapter; QueueAdapter = queueAdapter; TrackAdapter = trackAdapter; InvokeHandler = invokeHandler; BaseStoragePath = baseStoragePath; CurrentVersion = currentVersion; }
public Task InitializePullingAgents( string streamProviderName, StreamQueueBalancerType balancerType, StreamPubSubType pubSubType, IQueueAdapterFactory adapterFactory, IQueueAdapter queueAdapter, TimeSpan getQueueMsgsTimerPeriod, TimeSpan initQueueTimeout, TimeSpan maxEventDeliveryTime) { IStreamQueueBalancer queueBalancer = StreamQueueBalancerFactory.Create( balancerType, streamProviderName, Silo.CurrentSilo.LocalSiloStatusOracle, Silo.CurrentSilo.OrleansConfig, this, adapterFactory.GetStreamQueueMapper()); var managerId = GrainId.NewSystemTargetGrainIdByTypeCode(Constants.PULLING_AGENTS_MANAGER_SYSTEM_TARGET_TYPE_CODE); var manager = new PersistentStreamPullingManager(managerId, streamProviderName, this, this.PubSub(pubSubType), adapterFactory, queueBalancer, getQueueMsgsTimerPeriod, initQueueTimeout, maxEventDeliveryTime); this.RegisterSystemTarget(manager); // Init the manager only after it was registered locally. pullingAgentManager = manager.AsReference <IPersistentStreamPullingManager>(); // Need to call it as a grain reference though. return(pullingAgentManager.Initialize(queueAdapter.AsImmutable())); }
public QueueAdapter([NotNull] TQueue tQueue) { _adapter = tQueue switch { IDeque <T> deque => new DequeProxy(deque), ICircularBuffer <T> buffer => new CircularBufferProxy(buffer), IHeap <T> heap => new HeapProxy(heap), Queue <T> queue => new QueueProxy(queue), IQueue <T> queue => new QueueProxy(queue), Stack <T> stack => new StackProxy(stack), IStack <T> stack => new StackProxy(stack), ConcurrentQueue <T> queue => new ConcurrentQueueProxy(queue), ConcurrentStack <T> stack => new ConcurrentStackProxy(stack), BlockingCollection <T> blockingCollection => new BlockingCollectionProxy(blockingCollection), LinkedList <T> linkedList => new LinkedListProxy(linkedList), List <T> list => new ListProxy(list), _ => throw new NotSupportedException() }; Queue = tQueue; }
public void Init(IProviderConfiguration config, string providerName, Logger logger, IServiceProvider serviceProvider) { if (config == null) { throw new ArgumentNullException(nameof(config)); } if (logger == null) { throw new ArgumentNullException(nameof(logger)); } if (string.IsNullOrEmpty(providerName)) { throw new ArgumentNullException(nameof(providerName)); } _options = new RabbitMqStreamProviderOptions(config); _providerName = providerName; _cache = new ConcurrentQueueAdapterCache(_options.CacheSize); _mapper = new HashRingBasedStreamQueueMapper(_options.NumberOfQueues, _options.QueueNamePrefix); _failureHandler = Task.FromResult <IStreamFailureHandler>(new NoOpStreamDeliveryFailureHandler(false)); _adapter = new RabbitMqAdapter(_options, serviceProvider.GetRequiredService <SerializationManager>(), _mapper, _providerName, logger); }
public async Task Initialize(Immutable <IQueueAdapter> qAdapter) { if (qAdapter.Value == null) { throw new ArgumentNullException("qAdapter", "Init: queueAdapter should not be null"); } Log(ErrorCode.PersistentStreamPullingManager_02, "Init."); // Remove cast once we cleanup queueAdapter = qAdapter.Value; await this.queueBalancer.Initialize(this.adapterFactory.GetStreamQueueMapper()); queueBalancer.SubscribeToQueueDistributionChangeEvents(this); List <QueueId> myQueues = queueBalancer.GetMyQueues().ToList(); Log(ErrorCode.PersistentStreamPullingManager_03, String.Format("Initialize: I am now responsible for {0} queues: {1}.", myQueues.Count, PrintQueues(myQueues))); queuePrintTimer = this.RegisterTimer(AsyncTimerCallback, null, QUEUES_PRINT_PERIOD, QUEUES_PRINT_PERIOD); managerState = RunState.Initialized; }
public Task Initialize(Immutable <IQueueAdapter> qAdapter) { if (qAdapter.Value == null) { throw new ArgumentNullException("qAdapter", "Init: queueAdapter should not be null"); } logger.Info((int)ErrorCode.PersistentStreamPullingManager_02, "Init."); // Remove cast once we cleanup queueAdapter = qAdapter.Value; var meAsQueueBalanceListener = this.AsReference <IStreamQueueBalanceListener>(); queueBalancer.SubscribeToQueueDistributionChangeEvents(meAsQueueBalanceListener); List <QueueId> myQueues = queueBalancer.GetMyQueues().ToList(); logger.Info((int)ErrorCode.PersistentStreamPullingManager_03, PrintMyState(myQueues)); return(AddNewQueues(myQueues, true)); }
public Task Initialize(Immutable <IQueueAdapter> qAdapter) { if (qAdapter.Value == null) { throw new ArgumentNullException("qAdapter", "Init: queueAdapter should not be null"); } logger.Info((int)ErrorCode.PersistentStreamPullingManager_02, "Init."); // Remove cast once we cleanup queueAdapter = qAdapter.Value; var meAsQueueBalanceListener = this.AsReference <IStreamQueueBalanceListener>(); queueBalancer.SubscribeToQueueDistributionChangeEvents(meAsQueueBalanceListener); List <QueueId> myQueues = queueBalancer.GetMyQueues().ToList(); logger.Info((int)ErrorCode.PersistentStreamPullingManager_03, String.Format("I am now responsible for {0} queues: {1}.", myQueues.Count, PrintQueues(myQueues))); managerState = PersistentStreamProviderState.Initialized; return(TaskDone.Done); }
public Task Initialize(Immutable<IQueueAdapter> qAdapter) { if (qAdapter.Value == null) throw new ArgumentNullException("qAdapter", "Init: queueAdapter should not be null"); logger.Info((int)ErrorCode.PersistentStreamPullingManager_02, "Init."); // Remove cast once we cleanup queueAdapter = qAdapter.Value; var meAsQueueBalanceListener = this.AsReference<IStreamQueueBalanceListener>(); queueBalancer.SubscribeToQueueDistributionChangeEvents(meAsQueueBalanceListener); List<QueueId> myQueues = queueBalancer.GetMyQueues().ToList(); logger.Info((int)ErrorCode.PersistentStreamPullingManager_03, PrintMyState(myQueues)); return AddNewQueues(myQueues, true); }
public Task Initialize(Immutable<IQueueAdapter> qAdapter) { if (qAdapter.Value == null) throw new ArgumentNullException("qAdapter", "Init: queueAdapter should not be null"); Log(ErrorCode.PersistentStreamPullingManager_02, "Init."); // Remove cast once we cleanup queueAdapter = qAdapter.Value; var meAsQueueBalanceListener = this.AsReference<IStreamQueueBalanceListener>(); queueBalancer.SubscribeToQueueDistributionChangeEvents(meAsQueueBalanceListener); List<QueueId> myQueues = queueBalancer.GetMyQueues().ToList(); Log(ErrorCode.PersistentStreamPullingManager_03, String.Format("Initialize: I am now responsible for {0} queues: {1}.", myQueues.Count, PrintQueues(myQueues))); managerState = PersistentStreamProviderState.Initialized; return TaskDone.Done; }
/// <summary> /// Take responsibility for a new queues that was assigned to me via a new range. /// We first store the new queue in our internal data structure, try to initialize it and start a pumping timer. /// ERROR HANDLING: /// The resposibility to handle initializatoion and shutdown failures is inside the INewQueueAdapterReceiver code. /// The agent will call Initialize once and log an error. It will not call initiliaze again. /// The receiver itself may attempt later to recover from this error and do initialization again. /// The agent will assume initialization has succeeded and will subsequently start calling pumping receive. /// Same applies to shutdown. /// </summary> /// <param name="qAdapter"></param> /// <param name="queueAdapterCache"></param> /// <param name="failureHandler"></param> /// <returns></returns> public async Task Initialize(Immutable<IQueueAdapter> qAdapter, Immutable<IQueueAdapterCache> queueAdapterCache, Immutable<IStreamFailureHandler> failureHandler) { if (qAdapter.Value == null) throw new ArgumentNullException("qAdapter", "Init: queueAdapter should not be null"); if (failureHandler.Value == null) throw new ArgumentNullException("failureHandler", "Init: streamDeliveryFailureHandler should not be null"); logger.Info((int)ErrorCode.PersistentStreamPullingAgent_02, "Init of {0} {1} on silo {2} for queue {3}.", GetType().Name, GrainId.ToDetailedString(), Silo, QueueId.ToStringWithHashCode()); // Remove cast once we cleanup queueAdapter = qAdapter.Value; streamFailureHandler = failureHandler.Value; lastTimeCleanedPubSubCache = DateTime.UtcNow; try { receiver = queueAdapter.CreateReceiver(QueueId); } catch (Exception exc) { logger.Error((int)ErrorCode.PersistentStreamPullingAgent_02, "Exception while calling IQueueAdapter.CreateNewReceiver.", exc); return; } try { if (queueAdapterCache.Value != null) { queueCache = queueAdapterCache.Value.CreateQueueCache(QueueId); } } catch (Exception exc) { logger.Error((int)ErrorCode.PersistentStreamPullingAgent_23, "Exception while calling IQueueAdapterCache.CreateQueueCache.", exc); return; } try { var task = OrleansTaskExtentions.SafeExecute(() => receiver.Initialize(config.InitQueueTimeout)); task = task.LogException(logger, ErrorCode.PersistentStreamPullingAgent_03, String.Format("QueueAdapterReceiver {0} failed to Initialize.", QueueId.ToStringWithHashCode())); await task; } catch { // Just ignore this exception and proceed as if Initialize has succeeded. // We already logged individual exceptions for individual calls to Initialize. No need to log again. } // Setup a reader for a new receiver. // Even if the receiver failed to initialise, treat it as OK and start pumping it. It's receiver responsibility to retry initialization. var randomTimerOffset = safeRandom.NextTimeSpan(config.GetQueueMsgsTimerPeriod); timer = base.RegisterTimer(AsyncTimerCallback, QueueId, randomTimerOffset, config.GetQueueMsgsTimerPeriod); logger.Info((int) ErrorCode.PersistentStreamPullingAgent_04, "Taking queue {0} under my responsibility.", QueueId.ToStringWithHashCode()); }
private void InitializeInternal(IQueueAdapter qAdapter, IQueueAdapterCache queueAdapterCache, IStreamFailureHandler failureHandler) { logger.Info(ErrorCode.PersistentStreamPullingAgent_02, "Init of {0} {1} on silo {2} for queue {3}.", GetType().Name, GrainId.ToDetailedString(), Silo, QueueId.ToStringWithHashCode()); // Remove cast once we cleanup queueAdapter = qAdapter; streamFailureHandler = failureHandler; lastTimeCleanedPubSubCache = DateTime.UtcNow; try { receiver = queueAdapter.CreateReceiver(QueueId); } catch (Exception exc) { logger.Error(ErrorCode.PersistentStreamPullingAgent_02, "Exception while calling IQueueAdapter.CreateNewReceiver.", exc); throw; } try { if (queueAdapterCache != null) { queueCache = queueAdapterCache.CreateQueueCache(QueueId); } } catch (Exception exc) { logger.Error(ErrorCode.PersistentStreamPullingAgent_23, "Exception while calling IQueueAdapterCache.CreateQueueCache.", exc); throw; } try { receiverInitTask = OrleansTaskExtentions.SafeExecute(() => receiver.Initialize(config.InitQueueTimeout)) .LogException(logger, ErrorCode.PersistentStreamPullingAgent_03, $"QueueAdapterReceiver {QueueId.ToStringWithHashCode()} failed to Initialize."); receiverInitTask.Ignore(); } catch { // Just ignore this exception and proceed as if Initialize has succeeded. // We already logged individual exceptions for individual calls to Initialize. No need to log again. } // Setup a reader for a new receiver. // Even if the receiver failed to initialise, treat it as OK and start pumping it. It's receiver responsibility to retry initialization. var randomTimerOffset = safeRandom.NextTimeSpan(config.GetQueueMsgsTimerPeriod); timer = RegisterTimer(AsyncTimerCallback, QueueId, randomTimerOffset, config.GetQueueMsgsTimerPeriod); logger.Info((int)ErrorCode.PersistentStreamPullingAgent_04, "Taking queue {0} under my responsibility.", QueueId.ToStringWithHashCode()); }