public QueueConsumer(string queueName, int maximumThreads) { MaximumThreads = maximumThreads; HostId = ConfigSource.GetAppSetting("DQueue.HostId"); QueueName = queueName ?? QueueNameGenerator.GetQueueName <TMessage>(); Timeout = ConfigSource.FirstAppSetting("DQueue.ConsumerTimeout", "ConsumerTimeout").AsNullableTimeSpan(); if (string.IsNullOrWhiteSpace(HostId)) { throw new ArgumentNullException("HostId"); } if (string.IsNullOrWhiteSpace(QueueName)) { throw new ArgumentNullException("queueName"); } if (MaximumThreads < 1) { throw new ArgumentOutOfRangeException("maximumThreads"); } _provider = Constants.DefaultProvider; _cts = new CancellationTokenSource(); _messageHandlers = new List <Action <DispatchContext <TMessage> > >(); _timeoutHandlers = new List <Action <DispatchContext <TMessage> > >(); _completeHandlers = new List <Action <DispatchContext <TMessage> > >(); }
public async Task PublishEvent(string eventName, string eventKey, object eventData, DateTime?effectiveDate = null) { if (_shutdown) { throw new Exception("Host is not running"); } Logger.LogDebug("Creating event {0} {1}", eventName, eventKey); Event evt = new Event(); if (effectiveDate.HasValue) { evt.EventTime = effectiveDate.Value.ToUniversalTime(); } else { evt.EventTime = DateTime.Now.ToUniversalTime(); } evt.EventData = eventData; evt.EventKey = eventKey; evt.EventName = eventName; evt.IsProcessed = false; string eventId = await PersistenceStore.CreateEvent(evt); await QueueProvider.QueueWork(eventId, QueueType.Event); }
public void Start() { _shutdown = false; QueueProvider.Start().Wait(); LockProvider.Start().Wait(); for (int i = 0; i < Options.ThreadCount; i++) { Logger.LogInformation("Starting worker thread #{0}", i); IWorkflowThread thread = _serviceProvider.GetService <IWorkflowThread>(); _workers.Add(thread); thread.Start(); } Logger.LogInformation("Starting publish thread"); IEventThread pubThread = _serviceProvider.GetService <IEventThread>(); _workers.Add(pubThread); pubThread.Start(); Logger.LogInformation("Starting poller"); IRunnablePoller poller = _serviceProvider.GetService <IRunnablePoller>(); _workers.Add(poller); poller.Start(); }
private async Task <bool> SeedSubscription(Event evt, EventSubscription sub, List <string> toQueue, CancellationToken cancellationToken) { foreach (var eventId in await _eventRepository.GetEvents(sub.EventName, sub.EventKey, sub.SubscribeAsOf)) { if (eventId == evt.Id) { continue; } var siblingEvent = await _eventRepository.GetEvent(eventId); if ((!siblingEvent.IsProcessed) && (siblingEvent.EventTime < evt.EventTime)) { await QueueProvider.QueueWork(eventId, QueueType.Event); return(false); } if (!siblingEvent.IsProcessed) { toQueue.Add(siblingEvent.Id); } } if (!await _lockProvider.AcquireLock(sub.WorkflowId, cancellationToken)) { Logger.LogInformation("Workflow locked {0}", sub.WorkflowId); return(false); } try { var workflow = await _workflowRepository.GetWorkflowInstance(sub.WorkflowId); var pointers = workflow.ExecutionPointers.Where(p => p.EventName == sub.EventName && p.EventKey == sub.EventKey && !p.EventPublished && p.EndTime == null); foreach (var p in pointers) { p.EventData = evt.EventData; p.EventPublished = true; p.Active = true; } workflow.NextExecution = 0; await _workflowRepository.PersistWorkflow(workflow); await _subscriptionRepository.TerminateSubscription(sub.Id); return(true); } catch (Exception ex) { Logger.LogError(ex, ex.Message); return(false); } finally { await _lockProvider.ReleaseLock(sub.WorkflowId); await QueueProvider.QueueWork(sub.WorkflowId, QueueType.Workflow); } }
public static IQueueProvider CreateProvider(QueueProvider provider, bool singleton = false) { if (provider == QueueProvider.Configured) { QueueProvider outProvider; var strProvider = ConfigSource.FirstAppSetting("DQueue.Provider", "QueueProvider"); if (Enum.TryParse <QueueProvider>(strProvider, true, out outProvider)) { provider = outProvider; } else { throw new ArgumentException("Can not support queue provider: " + strProvider); } } if (provider == QueueProvider.Redis) { return(new RedisProvider()); } if (provider == QueueProvider.RabbitMQ) { return(new RabbitMQProvider()); } if (provider == QueueProvider.AspNet) { return(new AspNetProvider()); } throw new ArgumentException("Can not support queue provider: " + provider.ToString()); }
public async Task <bool> ResumeWorkflow(string workflowId) { if (LockProvider.AcquireLock(workflowId).Result) { bool requeue = false; try { var wf = await PersistenceStore.GetWorkflowInstance(workflowId); if (wf.Status == WorkflowStatus.Suspended) { wf.Status = WorkflowStatus.Runnable; await PersistenceStore.PersistWorkflow(wf); requeue = true; return(true); } return(false); } finally { await LockProvider.ReleaseLock(workflowId); if (requeue) { await QueueProvider.QueueForProcessing(workflowId); } } } return(false); }
public void DoesQueueExist_works() { var config = this.AwsConfig; var provider = new QueueProvider(config); var exists = provider.DoesQueueExist("3557b2d8a3264b1aa855f2a11557089c"); Assert.True(exists); }
/// <summary> /// Worker thread body /// </summary> private void RunWorkflows() { IWorkflowExecutor workflowExecutor = _serviceProvider.GetService <IWorkflowExecutor>(); IPersistenceProvider persistenceStore = _serviceProvider.GetService <IPersistenceProvider>(); while (!_shutdown) { try { var workflowId = QueueProvider.DequeueForProcessing().Result; if (workflowId != null) { try { if (LockProvider.AcquireLock(workflowId).Result) { WorkflowInstance workflow = null; try { workflow = persistenceStore.GetWorkflowInstance(workflowId).Result; if (workflow.Status == WorkflowStatus.Runnable) { workflowExecutor.Execute(workflow, persistenceStore, Options); } } finally { LockProvider.ReleaseLock(workflowId).Wait(); if (workflow != null) { if ((workflow.Status == WorkflowStatus.Runnable) && workflow.NextExecution.HasValue && workflow.NextExecution.Value < DateTime.Now.ToUniversalTime().Ticks) { QueueProvider.QueueForProcessing(workflowId); } } } } else { Logger.LogInformation("Workflow locked {0}", workflowId); } } catch (Exception ex) { Logger.LogError(ex.Message); } } else { Thread.Sleep(Options.IdleTime); //no work } } catch (Exception ex) { Logger.LogError(ex.Message); } } }
public void CalculateQueueUrl_correctly_joins_parts() { var provider = new QueueProvider(this.AwsConfig); var expected = "foo/bar"; var actual = provider.CalculateQueueUrl("foo/", "bar"); // Assert Assert.That(actual, Is.EqualTo(expected)); }
static void Main(string[] args) { ILessonRepository repository = new LessonRepository(); ILessonQueueProvider queueProvider = new QueueProvider(); ILessonJob job = new LessonProcessor(repository); Task t = ProcessAsync(queueProvider, job); t.Wait(); }
/// <summary> /// Poll the persistence store for workflows ready to run. /// Poll the persistence store for stashed unpublished events /// </summary> private void PollRunnables(object target) { try { if (LockProvider.AcquireLock("poll runnables").Result) { try { Logger.LogInformation("Polling for runnable workflows"); IPersistenceProvider persistenceStore = _serviceProvider.GetService <IPersistenceProvider>(); var runnables = persistenceStore.GetRunnableInstances().Result; foreach (var item in runnables) { Logger.LogDebug("Got runnable instance {0}", item); QueueProvider.QueueForProcessing(item); } } finally { LockProvider.ReleaseLock("poll runnables").Wait(); } } } catch (Exception ex) { Logger.LogError(ex.Message); } try { if (LockProvider.AcquireLock("unpublished events").Result) { try { Logger.LogInformation("Polling for unpublished events"); IPersistenceProvider persistenceStore = _serviceProvider.GetService <IPersistenceProvider>(); var events = persistenceStore.GetUnpublishedEvents().Result.ToList(); foreach (var item in events) { Logger.LogDebug("Got unpublished event {0} {1}", item.EventName, item.EventKey); QueueProvider.QueueForPublishing(item).Wait(); persistenceStore.RemoveUnpublishedEvent(item.Id).Wait(); } } finally { LockProvider.ReleaseLock("unpublished events").Wait(); } } } catch (Exception ex) { Logger.LogError(ex.Message); } }
protected override async Task ProcessItem(string itemId, CancellationToken cancellationToken) { if (!await _lockProvider.AcquireLock(itemId, cancellationToken)) { Logger.LogInformation("Workflow locked {0}", itemId); return; } WorkflowInstance workflow = null; WorkflowExecutorResult result = null; try { cancellationToken.ThrowIfCancellationRequested(); workflow = await _persistenceStore.GetWorkflowInstance(itemId, cancellationToken); if (workflow.Status == WorkflowStatus.Runnable) { try { result = await _executor.Execute(workflow, cancellationToken); } finally { await _persistenceStore.PersistWorkflow(workflow, cancellationToken); await QueueProvider.QueueWork(itemId, QueueType.Index); _greylist.Remove($"wf:{itemId}"); } } } finally { await _lockProvider.ReleaseLock(itemId); if ((workflow != null) && (result != null)) { foreach (var sub in result.Subscriptions) { await SubscribeEvent(sub, _persistenceStore, cancellationToken); } await _persistenceStore.PersistErrors(result.Errors, cancellationToken); var readAheadTicks = _datetimeProvider.UtcNow.Add(Options.PollInterval).Ticks; if ((workflow.Status == WorkflowStatus.Runnable) && workflow.NextExecution.HasValue && workflow.NextExecution.Value < readAheadTicks) { new Task(() => FutureQueue(workflow, cancellationToken)).Start(); } } } }
static void Main(string[] args) { var config = AwsConfig; var provider = new QueueProvider(config); var daffodil = provider.ReadMessage("daffodils"); // persist to SimpleDB var simpleDbProvider = new SimpleDbProvider(config); simpleDbProvider.Persist(daffodil); }
public static IQueuePublisher <T> CreateGooglePublisher <T>(QueueProvider provider, QueueSetting queueSetting, string topicName) { IQueuePublisher <T> publisher = default; switch (provider) { case QueueProvider.GOOGLE: publisher = new GoogleQueuePublisher <T>(queueSetting, topicName); break; } return(publisher); }
public static IQueueSubscriber CreateGoogleSubscriber(QueueProvider provider, QueueSetting queueSetting, string subscriptionName, Action <string> handler) { IQueueSubscriber subscriber = default; switch (provider) { case QueueProvider.GOOGLE: subscriber = new GoogleQueueSubscriber(queueSetting, subscriptionName, handler); break; } return(subscriber); }
public void CreateQueueIfNecessary_does_not_throw_error() { // Arrange var config = this.AwsConfig; var provider = new QueueProvider(config); var queueName = Guid.NewGuid().ToString().Replace("-", ""); // Act provider.CreateQueueIfNecessary(queueName); // Assert }
/// <summary> /// Constructor for the RabbitMQ Consumer Queue /// /// Listens on MittoMain /// /// ToDo: Make the sender Queue optional when creating a queue /// </summary> public Consumer(RabbitMQParams pParams) { RequestManager = new RequestManager(); QueueProvider = new QueueProvider(pParams); MainQueue = QueueProvider.GetReaderQueue(QueueType.Main, "Mitto.Main", true); MainQueue.Rx += MainQueue_Rx; ConsumerQueue = QueueProvider.GetReaderQueue(QueueType.Consumer, ID, false); ConsumerQueue.Rx += ConsumerQueue_Rx; }
protected override async Task ProcessItem(string itemId, CancellationToken cancellationToken) { try { var workflow = await FetchWorkflow(itemId); WorkflowActivity.Enrich(workflow, "index"); await _searchIndex.IndexWorkflow(workflow); lock (_errorCounts) { _errorCounts.Remove(itemId); } } catch (Exception e) { Logger.LogWarning(default(EventId), $"Error indexing workfow - {itemId} - {e.Message}"); var errCount = 0; lock (_errorCounts) { if (!_errorCounts.ContainsKey(itemId)) { _errorCounts.Add(itemId, 0); } _errorCounts[itemId]++; errCount = _errorCounts[itemId]; } if (errCount < 5) { await QueueProvider.QueueWork(itemId, Queue); return; } if (errCount < 20) { await Task.Delay(TimeSpan.FromSeconds(10)); await QueueProvider.QueueWork(itemId, Queue); return; } lock (_errorCounts) { _errorCounts.Remove(itemId); } Logger.LogError(default(EventId), e, $"Unable to index workfow - {itemId} - {e.Message}"); } }
public async Task It_Dispatches_A_Message() { // Arrange. var id = Guid.NewGuid(); var message = Message.WithContent("Hello, worldington.").WithId(id); // Act. var response = await QueueManager.Dispatch(message); // Assert. Assert.Equal(id, response.MessageId); Assert.True(QueueProvider.HasMessage(id)); }
public void CreateQueue_creates_queue_without_error() { // Arrange var config = this.AwsConfig; var provider = new QueueProvider(config); var queueName = Guid.NewGuid().ToString().Replace("-", ""); // Act provider.CreateQueue(queueName); // Assert }
private async Task SubscribeEvent(EventSubscription subscription, IPersistenceProvider persistenceStore, CancellationToken cancellationToken) { //TODO: move to own class Logger.LogDebug("Subscribing to event {0} {1} for workflow {2} step {3}", subscription.EventName, subscription.EventKey, subscription.WorkflowId, subscription.StepId); await persistenceStore.CreateEventSubscription(subscription, cancellationToken); if (subscription.EventName != Event.EventTypeActivity) { var events = await persistenceStore.GetEvents(subscription.EventName, subscription.EventKey, subscription.SubscribeAsOf, cancellationToken); foreach (var evt in events) { var eventKey = $"evt:{evt}"; bool acquiredLock = false; try { acquiredLock = await _lockProvider.AcquireLock(eventKey, cancellationToken); int attempt = 0; while (!acquiredLock && attempt < 10) { await Task.Delay(Options.IdleTime, cancellationToken); acquiredLock = await _lockProvider.AcquireLock(eventKey, cancellationToken); attempt++; } if (!acquiredLock) { Logger.LogWarning($"Failed to lock {evt}"); } else { _greylist.Remove(eventKey); await persistenceStore.MarkEventUnprocessed(evt, cancellationToken); await QueueProvider.QueueWork(evt, QueueType.Event); } } finally { if (acquiredLock) { await _lockProvider.ReleaseLock(eventKey); } } } } }
private async Task StashUnpublishedEvents() { if (!_shutdown) { var pub = await QueueProvider.DequeueForPublishing(); while (pub != null) { await PersistenceStore.CreateUnpublishedEvent(pub); pub = await QueueProvider.DequeueForPublishing(); } } }
public void Start() { _shutdown = false; PersistenceStore.EnsureStoreExists(); QueueProvider.Start().Wait(); LockProvider.Start().Wait(); Logger.LogInformation("Starting backgroud tasks"); foreach (var task in _backgroundTasks) { task.Start(); } }
public void Start() { QueueCache = new QueueProvider(new RabbitMQParams() { Hostname = "test.crazyzone.be" }); ReaderQueue = QueueCache.GetReaderQueue(QueueType.SubscriptionMain, ID, false); ReaderQueue.Rx += ObjMainReader_Rx; //ServiceQueue = QueueCache.GetReaderQueue(QueueType.SubscriptionMain, $"Mitto.Subscription.Main.{ID}", true); //var objSubscriptionConsumer = new ReaderQueue(QueueType.Subscription, "Mitto.Subscription." + Guid.NewGuid().ToString(), false); //var objMainWriter = new SenderQueue(QueueType.Subscription, "Mitto.Subscription.Main", true); }
public ActionResult Data(string id, string data) { var config = this.AwsConfig; var queueProvider = new QueueProvider(config); var daffodil = new Daffodil { Id = id, Data = data, }; queueProvider.SendMessage("daffodils", daffodil); return(Content("success")); }
/// <summary> /// Gets the hash code /// </summary> /// <returns>Hash code</returns> public override int GetHashCode() { unchecked // Overflow is fine, just wrap { var hashCode = 41; // Suitable nullity checks etc, of course :) if (Name != null) hashCode = hashCode * 59 + Name.GetHashCode(); if (Title != null) hashCode = hashCode * 59 + Title.GetHashCode(); if (Details != null) hashCode = hashCode * 59 + Details.GetHashCode(); if (Enabled != null) hashCode = hashCode * 59 + Enabled.GetHashCode(); if (ServiceName != null) hashCode = hashCode * 59 + ServiceName.GetHashCode(); if (LogLevel != null) hashCode = hashCode * 59 + LogLevel.GetHashCode(); if (AllowedRoots != null) hashCode = hashCode * 59 + AllowedRoots.GetHashCode(); if (QueueProcessingEnabled != null) hashCode = hashCode * 59 + QueueProcessingEnabled.GetHashCode(); if (PackageImporterEndpoints != null) hashCode = hashCode * 59 + PackageImporterEndpoints.GetHashCode(); if (PassiveQueues != null) hashCode = hashCode * 59 + PassiveQueues.GetHashCode(); if (PriorityQueues != null) hashCode = hashCode * 59 + PriorityQueues.GetHashCode(); if (RetryStrategy != null) hashCode = hashCode * 59 + RetryStrategy.GetHashCode(); if (RetryAttempts != null) hashCode = hashCode * 59 + RetryAttempts.GetHashCode(); if (RequestAuthorizationStrategyTarget != null) hashCode = hashCode * 59 + RequestAuthorizationStrategyTarget.GetHashCode(); if (TransportSecretProviderTarget != null) hashCode = hashCode * 59 + TransportSecretProviderTarget.GetHashCode(); if (PackageBuilderTarget != null) hashCode = hashCode * 59 + PackageBuilderTarget.GetHashCode(); if (TriggersTarget != null) hashCode = hashCode * 59 + TriggersTarget.GetHashCode(); if (QueueProvider != null) hashCode = hashCode * 59 + QueueProvider.GetHashCode(); if (AsyncDelivery != null) hashCode = hashCode * 59 + AsyncDelivery.GetHashCode(); if (HttpConnTimeout != null) hashCode = hashCode * 59 + HttpConnTimeout.GetHashCode(); return hashCode; } }
public void Stop() { _shutdown = true; Logger.LogInformation("Stopping background tasks"); foreach (var th in _backgroundTasks) { th.Stop(); } Logger.LogInformation("Worker tasks stopped"); QueueProvider.Stop(); LockProvider.Stop(); }
public async Task It_Continues_As_Normal_When_Message_Does_Not_Exist() { // Arrange. var id = Guid.NewGuid(); // Act. await QueueManager.Delete(new MemoryDeletable { Queue = "default", MessageId = id.ToString() }); // Assert. Assert.False(QueueProvider.HasMessage(id)); }
public static IQueueProvider GetSingleton(QueueProvider provider) { if (!_singletons.ContainsKey(provider)) { lock (typeof(QueueProviderFactory)) { if (!_singletons.ContainsKey(provider)) { _singletons[provider] = CreateProvider(provider); } } } return(_singletons[provider]); }
private async Task SubscribeEvent(EventSubscription subscription, IPersistenceProvider persistenceStore) { //TODO: move to own class Logger.LogDebug("Subscribing to event {0} {1} for workflow {2} step {3}", subscription.EventName, subscription.EventKey, subscription.WorkflowId, subscription.StepId); await persistenceStore.CreateEventSubscription(subscription); var events = await persistenceStore.GetEvents(subscription.EventName, subscription.EventKey, subscription.SubscribeAsOf); foreach (var evt in events) { await persistenceStore.MarkEventUnprocessed(evt); await QueueProvider.QueueWork(evt, QueueType.Event); } }
private void ReadQueueProvidersConfiguration() { XmlReaderSettings settings; XmlReader config; QueueProvider queueProvider; //string key, value; string enabled, def; settings = new XmlReaderSettings(); settings.Schemas.Add(null, Constants.QUEUE_PROVIDERS_CONFIG_SCHEMA); settings.ValidationType = ValidationType.Schema; config = XmlReader.Create(Constants.QUEUE_PROVIDERS_CONFIG, settings); config.ReadStartElement("usmsgw"); config.ReadStartElement("queueProviders"); while (config.ReadToFollowing("queueProvider")) { def = config.GetAttribute("default"); enabled = config.GetAttribute("enabled"); config.Read(); queueProvider = new QueueProvider { Name = config.ReadElementString("name"), AssemblyName = config.ReadElementString("assembly"), ClassName = config.ReadElementString("class"), Enabled = IsFeatureEnabled(enabled) }; _queueProviders.Add(queueProvider.Name.Trim().ToLower(), queueProvider); if (IsFeatureEnabled(def)) { if (_defaultQueueProvider != null) throw ProviderConfigurationException.MultipleDefaultProviders(EnProviderType.QueueProvider); _defaultQueueProvider = queueProvider; } /* /// /// Configuration /// config.ReadToFollowing("configuration"); config.ReadToFollowing("set"); do { key = config.GetAttribute("key"); value = config.GetAttribute("value"); if ((string.IsNullOrEmpty(key) == false) && (string.IsNullOrEmpty(value) == false)) queueProvider.Configuration.Settings[key] = value; } while (config.ReadToNextSibling("set")); */ } if (_defaultQueueProvider == null) throw ProviderConfigurationException.MissingDefaultProvider(EnProviderType.QueueProvider); }