public QueueConsumer(string queueName, int maximumThreads)
{
MaximumThreads = maximumThreads;
HostId = ConfigSource.GetAppSetting("DQueue.HostId");
QueueName = queueName ?? QueueNameGenerator.GetQueueName <TMessage>();
Timeout = ConfigSource.FirstAppSetting("DQueue.ConsumerTimeout", "ConsumerTimeout").AsNullableTimeSpan();
if (string.IsNullOrWhiteSpace(HostId))
{
throw new ArgumentNullException("HostId");
}
if (string.IsNullOrWhiteSpace(QueueName))
{
throw new ArgumentNullException("queueName");
}
if (MaximumThreads < 1)
{
throw new ArgumentOutOfRangeException("maximumThreads");
}
_provider = Constants.DefaultProvider;
_cts = new CancellationTokenSource();
_messageHandlers = new List <Action <DispatchContext <TMessage> > >();
_timeoutHandlers = new List <Action <DispatchContext <TMessage> > >();
_completeHandlers = new List <Action <DispatchContext <TMessage> > >();
}
public async Task PublishEvent(string eventName, string eventKey, object eventData, DateTime?effectiveDate = null)
{
if (_shutdown)
{
throw new Exception("Host is not running");
}
Logger.LogDebug("Creating event {0} {1}", eventName, eventKey);
Event evt = new Event();
if (effectiveDate.HasValue)
{
evt.EventTime = effectiveDate.Value.ToUniversalTime();
}
else
{
evt.EventTime = DateTime.Now.ToUniversalTime();
}
evt.EventData = eventData;
evt.EventKey = eventKey;
evt.EventName = eventName;
evt.IsProcessed = false;
string eventId = await PersistenceStore.CreateEvent(evt);
await QueueProvider.QueueWork(eventId, QueueType.Event);
}
public void Start()
{
_shutdown = false;
QueueProvider.Start().Wait();
LockProvider.Start().Wait();
for (int i = 0; i < Options.ThreadCount; i++)
{
Logger.LogInformation("Starting worker thread #{0}", i);
IWorkflowThread thread = _serviceProvider.GetService <IWorkflowThread>();
_workers.Add(thread);
thread.Start();
}
Logger.LogInformation("Starting publish thread");
IEventThread pubThread = _serviceProvider.GetService <IEventThread>();
_workers.Add(pubThread);
pubThread.Start();
Logger.LogInformation("Starting poller");
IRunnablePoller poller = _serviceProvider.GetService <IRunnablePoller>();
_workers.Add(poller);
poller.Start();
}
private async Task <bool> SeedSubscription(Event evt, EventSubscription sub, List <string> toQueue, CancellationToken cancellationToken)
{
foreach (var eventId in await _eventRepository.GetEvents(sub.EventName, sub.EventKey, sub.SubscribeAsOf))
{
if (eventId == evt.Id)
{
continue;
}
var siblingEvent = await _eventRepository.GetEvent(eventId);
if ((!siblingEvent.IsProcessed) && (siblingEvent.EventTime < evt.EventTime))
{
await QueueProvider.QueueWork(eventId, QueueType.Event);
return(false);
}
if (!siblingEvent.IsProcessed)
{
toQueue.Add(siblingEvent.Id);
}
}
if (!await _lockProvider.AcquireLock(sub.WorkflowId, cancellationToken))
{
Logger.LogInformation("Workflow locked {0}", sub.WorkflowId);
return(false);
}
try
{
var workflow = await _workflowRepository.GetWorkflowInstance(sub.WorkflowId);
var pointers = workflow.ExecutionPointers.Where(p => p.EventName == sub.EventName && p.EventKey == sub.EventKey && !p.EventPublished && p.EndTime == null);
foreach (var p in pointers)
{
p.EventData = evt.EventData;
p.EventPublished = true;
p.Active = true;
}
workflow.NextExecution = 0;
await _workflowRepository.PersistWorkflow(workflow);
await _subscriptionRepository.TerminateSubscription(sub.Id);
return(true);
}
catch (Exception ex)
{
Logger.LogError(ex, ex.Message);
return(false);
}
finally
{
await _lockProvider.ReleaseLock(sub.WorkflowId);
await QueueProvider.QueueWork(sub.WorkflowId, QueueType.Workflow);
}
}
public static IQueueProvider CreateProvider(QueueProvider provider, bool singleton = false)
{
if (provider == QueueProvider.Configured)
{
QueueProvider outProvider;
var strProvider = ConfigSource.FirstAppSetting("DQueue.Provider", "QueueProvider");
if (Enum.TryParse <QueueProvider>(strProvider, true, out outProvider))
{
provider = outProvider;
}
else
{
throw new ArgumentException("Can not support queue provider: " + strProvider);
}
}
if (provider == QueueProvider.Redis)
{
return(new RedisProvider());
}
if (provider == QueueProvider.RabbitMQ)
{
return(new RabbitMQProvider());
}
if (provider == QueueProvider.AspNet)
{
return(new AspNetProvider());
}
throw new ArgumentException("Can not support queue provider: " + provider.ToString());
}
public async Task <bool> ResumeWorkflow(string workflowId)
{
if (LockProvider.AcquireLock(workflowId).Result)
{
bool requeue = false;
try
{
var wf = await PersistenceStore.GetWorkflowInstance(workflowId);
if (wf.Status == WorkflowStatus.Suspended)
{
wf.Status = WorkflowStatus.Runnable;
await PersistenceStore.PersistWorkflow(wf);
requeue = true;
return(true);
}
return(false);
}
finally
{
await LockProvider.ReleaseLock(workflowId);
if (requeue)
{
await QueueProvider.QueueForProcessing(workflowId);
}
}
}
return(false);
}
public void DoesQueueExist_works()
{
var config = this.AwsConfig;
var provider = new QueueProvider(config);
var exists = provider.DoesQueueExist("3557b2d8a3264b1aa855f2a11557089c");
Assert.True(exists);
}
/// <summary>
/// Worker thread body
/// </summary>
private void RunWorkflows()
{
IWorkflowExecutor workflowExecutor = _serviceProvider.GetService <IWorkflowExecutor>();
IPersistenceProvider persistenceStore = _serviceProvider.GetService <IPersistenceProvider>();
while (!_shutdown)
{
try
{
var workflowId = QueueProvider.DequeueForProcessing().Result;
if (workflowId != null)
{
try
{
if (LockProvider.AcquireLock(workflowId).Result)
{
WorkflowInstance workflow = null;
try
{
workflow = persistenceStore.GetWorkflowInstance(workflowId).Result;
if (workflow.Status == WorkflowStatus.Runnable)
{
workflowExecutor.Execute(workflow, persistenceStore, Options);
}
}
finally
{
LockProvider.ReleaseLock(workflowId).Wait();
if (workflow != null)
{
if ((workflow.Status == WorkflowStatus.Runnable) && workflow.NextExecution.HasValue && workflow.NextExecution.Value < DateTime.Now.ToUniversalTime().Ticks)
{
QueueProvider.QueueForProcessing(workflowId);
}
}
}
}
else
{
Logger.LogInformation("Workflow locked {0}", workflowId);
}
}
catch (Exception ex)
{
Logger.LogError(ex.Message);
}
}
else
{
Thread.Sleep(Options.IdleTime); //no work
}
}
catch (Exception ex)
{
Logger.LogError(ex.Message);
}
}
}
public void CalculateQueueUrl_correctly_joins_parts()
{
var provider = new QueueProvider(this.AwsConfig);
var expected = "foo/bar";
var actual = provider.CalculateQueueUrl("foo/", "bar");
// Assert
Assert.That(actual, Is.EqualTo(expected));
}
static void Main(string[] args)
{
ILessonRepository repository = new LessonRepository();
ILessonQueueProvider queueProvider = new QueueProvider();
ILessonJob job = new LessonProcessor(repository);
Task t = ProcessAsync(queueProvider, job);
t.Wait();
}
/// <summary>
/// Poll the persistence store for workflows ready to run.
/// Poll the persistence store for stashed unpublished events
/// </summary>
private void PollRunnables(object target)
{
try
{
if (LockProvider.AcquireLock("poll runnables").Result)
{
try
{
Logger.LogInformation("Polling for runnable workflows");
IPersistenceProvider persistenceStore = _serviceProvider.GetService <IPersistenceProvider>();
var runnables = persistenceStore.GetRunnableInstances().Result;
foreach (var item in runnables)
{
Logger.LogDebug("Got runnable instance {0}", item);
QueueProvider.QueueForProcessing(item);
}
}
finally
{
LockProvider.ReleaseLock("poll runnables").Wait();
}
}
}
catch (Exception ex)
{
Logger.LogError(ex.Message);
}
try
{
if (LockProvider.AcquireLock("unpublished events").Result)
{
try
{
Logger.LogInformation("Polling for unpublished events");
IPersistenceProvider persistenceStore = _serviceProvider.GetService <IPersistenceProvider>();
var events = persistenceStore.GetUnpublishedEvents().Result.ToList();
foreach (var item in events)
{
Logger.LogDebug("Got unpublished event {0} {1}", item.EventName, item.EventKey);
QueueProvider.QueueForPublishing(item).Wait();
persistenceStore.RemoveUnpublishedEvent(item.Id).Wait();
}
}
finally
{
LockProvider.ReleaseLock("unpublished events").Wait();
}
}
}
catch (Exception ex)
{
Logger.LogError(ex.Message);
}
}
protected override async Task ProcessItem(string itemId, CancellationToken cancellationToken)
{
if (!await _lockProvider.AcquireLock(itemId, cancellationToken))
{
Logger.LogInformation("Workflow locked {0}", itemId);
return;
}
WorkflowInstance workflow = null;
WorkflowExecutorResult result = null;
try
{
cancellationToken.ThrowIfCancellationRequested();
workflow = await _persistenceStore.GetWorkflowInstance(itemId, cancellationToken);
if (workflow.Status == WorkflowStatus.Runnable)
{
try
{
result = await _executor.Execute(workflow, cancellationToken);
}
finally
{
await _persistenceStore.PersistWorkflow(workflow, cancellationToken);
await QueueProvider.QueueWork(itemId, QueueType.Index);
_greylist.Remove($"wf:{itemId}");
}
}
}
finally
{
await _lockProvider.ReleaseLock(itemId);
if ((workflow != null) && (result != null))
{
foreach (var sub in result.Subscriptions)
{
await SubscribeEvent(sub, _persistenceStore, cancellationToken);
}
await _persistenceStore.PersistErrors(result.Errors, cancellationToken);
var readAheadTicks = _datetimeProvider.UtcNow.Add(Options.PollInterval).Ticks;
if ((workflow.Status == WorkflowStatus.Runnable) && workflow.NextExecution.HasValue && workflow.NextExecution.Value < readAheadTicks)
{
new Task(() => FutureQueue(workflow, cancellationToken)).Start();
}
}
}
}
static void Main(string[] args)
{
var config = AwsConfig;
var provider = new QueueProvider(config);
var daffodil = provider.ReadMessage("daffodils");
// persist to SimpleDB
var simpleDbProvider = new SimpleDbProvider(config);
simpleDbProvider.Persist(daffodil);
}
public static IQueuePublisher <T> CreateGooglePublisher <T>(QueueProvider provider, QueueSetting queueSetting, string topicName)
{
IQueuePublisher <T> publisher = default;
switch (provider)
{
case QueueProvider.GOOGLE:
publisher = new GoogleQueuePublisher <T>(queueSetting, topicName);
break;
}
return(publisher);
}
public static IQueueSubscriber CreateGoogleSubscriber(QueueProvider provider, QueueSetting queueSetting, string subscriptionName, Action <string> handler)
{
IQueueSubscriber subscriber = default;
switch (provider)
{
case QueueProvider.GOOGLE:
subscriber = new GoogleQueueSubscriber(queueSetting, subscriptionName, handler);
break;
}
return(subscriber);
}
public void CreateQueueIfNecessary_does_not_throw_error()
{
// Arrange
var config = this.AwsConfig;
var provider = new QueueProvider(config);
var queueName = Guid.NewGuid().ToString().Replace("-", "");
// Act
provider.CreateQueueIfNecessary(queueName);
// Assert
}
/// <summary>
/// Constructor for the RabbitMQ Consumer Queue
///
/// Listens on MittoMain
///
/// ToDo: Make the sender Queue optional when creating a queue
/// </summary>
public Consumer(RabbitMQParams pParams)
{
RequestManager = new RequestManager();
QueueProvider = new QueueProvider(pParams);
MainQueue = QueueProvider.GetReaderQueue(QueueType.Main, "Mitto.Main", true);
MainQueue.Rx += MainQueue_Rx;
ConsumerQueue = QueueProvider.GetReaderQueue(QueueType.Consumer, ID, false);
ConsumerQueue.Rx += ConsumerQueue_Rx;
}
protected override async Task ProcessItem(string itemId, CancellationToken cancellationToken)
{
try
{
var workflow = await FetchWorkflow(itemId);
WorkflowActivity.Enrich(workflow, "index");
await _searchIndex.IndexWorkflow(workflow);
lock (_errorCounts)
{
_errorCounts.Remove(itemId);
}
}
catch (Exception e)
{
Logger.LogWarning(default(EventId), $"Error indexing workfow - {itemId} - {e.Message}");
var errCount = 0;
lock (_errorCounts)
{
if (!_errorCounts.ContainsKey(itemId))
{
_errorCounts.Add(itemId, 0);
}
_errorCounts[itemId]++;
errCount = _errorCounts[itemId];
}
if (errCount < 5)
{
await QueueProvider.QueueWork(itemId, Queue);
return;
}
if (errCount < 20)
{
await Task.Delay(TimeSpan.FromSeconds(10));
await QueueProvider.QueueWork(itemId, Queue);
return;
}
lock (_errorCounts)
{
_errorCounts.Remove(itemId);
}
Logger.LogError(default(EventId), e, $"Unable to index workfow - {itemId} - {e.Message}");
}
}
public async Task It_Dispatches_A_Message()
{
// Arrange.
var id = Guid.NewGuid();
var message = Message.WithContent("Hello, worldington.").WithId(id);
// Act.
var response = await QueueManager.Dispatch(message);
// Assert.
Assert.Equal(id, response.MessageId);
Assert.True(QueueProvider.HasMessage(id));
}
public void CreateQueue_creates_queue_without_error()
{
// Arrange
var config = this.AwsConfig;
var provider = new QueueProvider(config);
var queueName = Guid.NewGuid().ToString().Replace("-", "");
// Act
provider.CreateQueue(queueName);
// Assert
}
private async Task SubscribeEvent(EventSubscription subscription, IPersistenceProvider persistenceStore, CancellationToken cancellationToken)
{
//TODO: move to own class
Logger.LogDebug("Subscribing to event {0} {1} for workflow {2} step {3}", subscription.EventName, subscription.EventKey, subscription.WorkflowId, subscription.StepId);
await persistenceStore.CreateEventSubscription(subscription, cancellationToken);
if (subscription.EventName != Event.EventTypeActivity)
{
var events = await persistenceStore.GetEvents(subscription.EventName, subscription.EventKey, subscription.SubscribeAsOf, cancellationToken);
foreach (var evt in events)
{
var eventKey = $"evt:{evt}";
bool acquiredLock = false;
try
{
acquiredLock = await _lockProvider.AcquireLock(eventKey, cancellationToken);
int attempt = 0;
while (!acquiredLock && attempt < 10)
{
await Task.Delay(Options.IdleTime, cancellationToken);
acquiredLock = await _lockProvider.AcquireLock(eventKey, cancellationToken);
attempt++;
}
if (!acquiredLock)
{
Logger.LogWarning($"Failed to lock {evt}");
}
else
{
_greylist.Remove(eventKey);
await persistenceStore.MarkEventUnprocessed(evt, cancellationToken);
await QueueProvider.QueueWork(evt, QueueType.Event);
}
}
finally
{
if (acquiredLock)
{
await _lockProvider.ReleaseLock(eventKey);
}
}
}
}
}
private async Task StashUnpublishedEvents()
{
if (!_shutdown)
{
var pub = await QueueProvider.DequeueForPublishing();
while (pub != null)
{
await PersistenceStore.CreateUnpublishedEvent(pub);
pub = await QueueProvider.DequeueForPublishing();
}
}
}
public void Start()
{
_shutdown = false;
PersistenceStore.EnsureStoreExists();
QueueProvider.Start().Wait();
LockProvider.Start().Wait();
Logger.LogInformation("Starting backgroud tasks");
foreach (var task in _backgroundTasks)
{
task.Start();
}
}
public void Start()
{
QueueCache = new QueueProvider(new RabbitMQParams()
{
Hostname = "test.crazyzone.be"
});
ReaderQueue = QueueCache.GetReaderQueue(QueueType.SubscriptionMain, ID, false);
ReaderQueue.Rx += ObjMainReader_Rx;
//ServiceQueue = QueueCache.GetReaderQueue(QueueType.SubscriptionMain, $"Mitto.Subscription.Main.{ID}", true);
//var objSubscriptionConsumer = new ReaderQueue(QueueType.Subscription, "Mitto.Subscription." + Guid.NewGuid().ToString(), false);
//var objMainWriter = new SenderQueue(QueueType.Subscription, "Mitto.Subscription.Main", true);
}
public ActionResult Data(string id, string data)
{
var config = this.AwsConfig;
var queueProvider = new QueueProvider(config);
var daffodil = new Daffodil
{
Id = id,
Data = data,
};
queueProvider.SendMessage("daffodils", daffodil);
return(Content("success"));
}
/// <summary>
/// Gets the hash code
/// </summary>
/// <returns>Hash code</returns>
public override int GetHashCode()
{
unchecked // Overflow is fine, just wrap
{
var hashCode = 41;
// Suitable nullity checks etc, of course :)
if (Name != null)
hashCode = hashCode * 59 + Name.GetHashCode();
if (Title != null)
hashCode = hashCode * 59 + Title.GetHashCode();
if (Details != null)
hashCode = hashCode * 59 + Details.GetHashCode();
if (Enabled != null)
hashCode = hashCode * 59 + Enabled.GetHashCode();
if (ServiceName != null)
hashCode = hashCode * 59 + ServiceName.GetHashCode();
if (LogLevel != null)
hashCode = hashCode * 59 + LogLevel.GetHashCode();
if (AllowedRoots != null)
hashCode = hashCode * 59 + AllowedRoots.GetHashCode();
if (QueueProcessingEnabled != null)
hashCode = hashCode * 59 + QueueProcessingEnabled.GetHashCode();
if (PackageImporterEndpoints != null)
hashCode = hashCode * 59 + PackageImporterEndpoints.GetHashCode();
if (PassiveQueues != null)
hashCode = hashCode * 59 + PassiveQueues.GetHashCode();
if (PriorityQueues != null)
hashCode = hashCode * 59 + PriorityQueues.GetHashCode();
if (RetryStrategy != null)
hashCode = hashCode * 59 + RetryStrategy.GetHashCode();
if (RetryAttempts != null)
hashCode = hashCode * 59 + RetryAttempts.GetHashCode();
if (RequestAuthorizationStrategyTarget != null)
hashCode = hashCode * 59 + RequestAuthorizationStrategyTarget.GetHashCode();
if (TransportSecretProviderTarget != null)
hashCode = hashCode * 59 + TransportSecretProviderTarget.GetHashCode();
if (PackageBuilderTarget != null)
hashCode = hashCode * 59 + PackageBuilderTarget.GetHashCode();
if (TriggersTarget != null)
hashCode = hashCode * 59 + TriggersTarget.GetHashCode();
if (QueueProvider != null)
hashCode = hashCode * 59 + QueueProvider.GetHashCode();
if (AsyncDelivery != null)
hashCode = hashCode * 59 + AsyncDelivery.GetHashCode();
if (HttpConnTimeout != null)
hashCode = hashCode * 59 + HttpConnTimeout.GetHashCode();
return hashCode;
}
}
public void Stop()
{
_shutdown = true;
Logger.LogInformation("Stopping background tasks");
foreach (var th in _backgroundTasks)
{
th.Stop();
}
Logger.LogInformation("Worker tasks stopped");
QueueProvider.Stop();
LockProvider.Stop();
}
public async Task It_Continues_As_Normal_When_Message_Does_Not_Exist()
{
// Arrange.
var id = Guid.NewGuid();
// Act.
await QueueManager.Delete(new MemoryDeletable
{
Queue = "default",
MessageId = id.ToString()
});
// Assert.
Assert.False(QueueProvider.HasMessage(id));
}
public static IQueueProvider GetSingleton(QueueProvider provider)
{
if (!_singletons.ContainsKey(provider))
{
lock (typeof(QueueProviderFactory))
{
if (!_singletons.ContainsKey(provider))
{
_singletons[provider] = CreateProvider(provider);
}
}
}
return(_singletons[provider]);
}
private async Task SubscribeEvent(EventSubscription subscription, IPersistenceProvider persistenceStore)
{
//TODO: move to own class
Logger.LogDebug("Subscribing to event {0} {1} for workflow {2} step {3}", subscription.EventName, subscription.EventKey, subscription.WorkflowId, subscription.StepId);
await persistenceStore.CreateEventSubscription(subscription);
var events = await persistenceStore.GetEvents(subscription.EventName, subscription.EventKey, subscription.SubscribeAsOf);
foreach (var evt in events)
{
await persistenceStore.MarkEventUnprocessed(evt);
await QueueProvider.QueueWork(evt, QueueType.Event);
}
}
private void ReadQueueProvidersConfiguration()
{
XmlReaderSettings settings;
XmlReader config;
QueueProvider queueProvider;
//string key, value;
string enabled, def;
settings = new XmlReaderSettings();
settings.Schemas.Add(null, Constants.QUEUE_PROVIDERS_CONFIG_SCHEMA);
settings.ValidationType = ValidationType.Schema;
config = XmlReader.Create(Constants.QUEUE_PROVIDERS_CONFIG, settings);
config.ReadStartElement("usmsgw");
config.ReadStartElement("queueProviders");
while (config.ReadToFollowing("queueProvider"))
{
def = config.GetAttribute("default");
enabled = config.GetAttribute("enabled");
config.Read();
queueProvider = new QueueProvider
{
Name = config.ReadElementString("name"),
AssemblyName = config.ReadElementString("assembly"),
ClassName = config.ReadElementString("class"),
Enabled = IsFeatureEnabled(enabled)
};
_queueProviders.Add(queueProvider.Name.Trim().ToLower(), queueProvider);
if (IsFeatureEnabled(def))
{
if (_defaultQueueProvider != null)
throw ProviderConfigurationException.MultipleDefaultProviders(EnProviderType.QueueProvider);
_defaultQueueProvider = queueProvider;
}
/*
///
/// Configuration
///
config.ReadToFollowing("configuration");
config.ReadToFollowing("set");
do
{
key = config.GetAttribute("key");
value = config.GetAttribute("value");
if ((string.IsNullOrEmpty(key) == false) && (string.IsNullOrEmpty(value) == false))
queueProvider.Configuration.Settings[key] = value;
} while (config.ReadToNextSibling("set"));
*/
}
if (_defaultQueueProvider == null)
throw ProviderConfigurationException.MissingDefaultProvider(EnProviderType.QueueProvider);
}
